From e7f1649fe66300ebe1deb16a4240276ee2d5392a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 4 Nov 2021 07:05:24 +0000 Subject: [PATCH] A new CUE standard library for the Europa release Signed-off-by: Solomon Hykes --- europa/README.md | 120 ++++++++++ europa/fmt.sh | 3 + europa/search.sh | 3 + europa/stdlib/.gitignore | 2 + europa/stdlib/cue.mod/module.cue | 1 + europa/stdlib/dagger/engine/exec.cue | 64 ++++++ europa/stdlib/dagger/engine/fs.cue | 56 +++++ europa/stdlib/dagger/engine/git.cue | 19 ++ europa/stdlib/dagger/engine/image.cue | 88 ++++++++ europa/stdlib/dagger/engine/secret.cue | 6 + europa/stdlib/dagger/engine/service.cue | 6 + europa/stdlib/dagger/engine/stream.cue | 6 + europa/stdlib/dagger/plan.cue | 97 +++++++++ europa/stdlib/dagger/types.cue | 36 +++ europa/stdlib/dagger/utils.cue | 26 +++ europa/test.sh | 36 +++ europa/universe/alpine/alpine.cue | 36 +++ .../universe/alpine/tests/simple/simple.cue | 38 ++++ europa/universe/bash/bash.cue | 21 ++ europa/universe/cue.mod/module.cue | 1 + europa/universe/cue.mod/pkg/.gitignore | 2 + europa/universe/cue.mod/pkg/dagger.io | 1 + europa/universe/docker/build.cue | 97 +++++++++ europa/universe/docker/docker.cue | 205 ++++++++++++++++++ europa/universe/docker/test/bash/bash.cue | 15 ++ europa/universe/docker/test/build/build.cue | 82 +++++++ europa/universe/examples/README.md | 3 + europa/universe/examples/todoapp/base.cue | 28 +++ europa/universe/examples/todoapp/dev/dev.cue | 40 ++++ .../examples/todoapp/staging/staging.cue | 25 +++ europa/universe/git/git.cue | 8 + europa/universe/netlify/deploy.sh | 1 + europa/universe/netlify/deploy.sh.cue | 56 +++++ europa/universe/netlify/netlify.cue | 99 +++++++++ .../universe/netlify/test/simple/simple.cue | 9 + europa/universe/nginx/nginx.cue | 26 +++ europa/universe/python/python.cue | 24 ++ europa/universe/yarn/tests/simple/simple.cue | 11 + .../universe/yarn/tests/testdata/package.json | 11 + .../yarn/tests/testdata2/package.json | 12 + europa/universe/yarn/yarn.cue | 96 ++++++++ 41 files changed, 1516 insertions(+) create mode 100644 europa/README.md create mode 100755 europa/fmt.sh create mode 100755 europa/search.sh create mode 100644 europa/stdlib/.gitignore create mode 100644 europa/stdlib/cue.mod/module.cue create mode 100644 europa/stdlib/dagger/engine/exec.cue create mode 100644 europa/stdlib/dagger/engine/fs.cue create mode 100644 europa/stdlib/dagger/engine/git.cue create mode 100644 europa/stdlib/dagger/engine/image.cue create mode 100644 europa/stdlib/dagger/engine/secret.cue create mode 100644 europa/stdlib/dagger/engine/service.cue create mode 100644 europa/stdlib/dagger/engine/stream.cue create mode 100644 europa/stdlib/dagger/plan.cue create mode 100644 europa/stdlib/dagger/types.cue create mode 100644 europa/stdlib/dagger/utils.cue create mode 100755 europa/test.sh create mode 100644 europa/universe/alpine/alpine.cue create mode 100644 europa/universe/alpine/tests/simple/simple.cue create mode 100644 europa/universe/bash/bash.cue create mode 100644 europa/universe/cue.mod/module.cue create mode 100644 europa/universe/cue.mod/pkg/.gitignore create mode 120000 europa/universe/cue.mod/pkg/dagger.io create mode 100644 europa/universe/docker/build.cue create mode 100644 europa/universe/docker/docker.cue create mode 100644 europa/universe/docker/test/bash/bash.cue create mode 100644 europa/universe/docker/test/build/build.cue create mode 100644 europa/universe/examples/README.md create mode 100644 europa/universe/examples/todoapp/base.cue create mode 100644 europa/universe/examples/todoapp/dev/dev.cue create mode 100644 europa/universe/examples/todoapp/staging/staging.cue create mode 100644 europa/universe/git/git.cue create mode 120000 europa/universe/netlify/deploy.sh create mode 100644 europa/universe/netlify/deploy.sh.cue create mode 100644 europa/universe/netlify/netlify.cue create mode 100644 europa/universe/netlify/test/simple/simple.cue create mode 100644 europa/universe/nginx/nginx.cue create mode 100644 europa/universe/python/python.cue create mode 100644 europa/universe/yarn/tests/simple/simple.cue create mode 100644 europa/universe/yarn/tests/testdata/package.json create mode 100644 europa/universe/yarn/tests/testdata2/package.json create mode 100644 europa/universe/yarn/yarn.cue diff --git a/europa/README.md b/europa/README.md new file mode 100644 index 00000000..047c9b25 --- /dev/null +++ b/europa/README.md @@ -0,0 +1,120 @@ +# Europa release staging + +## About the europa/ directory + +This directory is a staging area for the upcoming Europa release. +It is intended for experimentation and review without requiring long-lived development branches. +Its contents MUST NOT BE USED by `dagger` or its build and release tooling. + +As part of the Europa release, this directory will be removed. + +## About the Europa release + +Europa is the codename of the final major release of Dagger before launch. +For more details on the Europa release, see the [Europa epic](https://github.com/dagger/dagger/issues/1088). + +## New CUE packages + +Europa introduces a new set of CUE packages for developers to use. These new packages are a complete, incompatible replacement for the pre-Europa packages. +* The bad news is that pre-Europa configurations will need to be manually adapted +* The good news is that Europa APIs are much better. So once ported to Europa, configurations will be shorter, easier to maintain, faster and more reliable (at least that's the goal!) + +We intend for Europa to be the last breaking update. Going forward, we will aim for 100% compatibility +whenever possible, and when that is not possible, a migration path that is as automated and painless +as possible. + +Starting with Europa, Dagger separates its Cue packages in two distinct namespaces: *stdlib* and *universe*. + +* The Dagger Universe is a catalog of reusable Cue packages, curated by Dagger but possibly authored by third parties. Most packages in Universe contain reusable actions; some may also contain entire configuration templates. +* The Dagger Stdlib are core packages shipped with the Dagger engine. + +| | *Stdlib* | *Universe* | +|----------|--------------|------| +| Import path | `dagger.io` | `universe.dagger.io` | +| Purpose | Access core Dagger features | Safely reuse code from the community | +| Author | Dagger team | Dagger community, curated by Dagger | +| Release cycle | Released with Dagger engine | Released continuously | +| Size | Small | Large | +| Growth rate | Grows slowly, with engine features | Grows fast, with community | + +### Dagger Core API + +*Import path: [`dagger.io/dagger`](./stdlib/dagger)* + +The Dagger Core API defines core types and utilities for programming Dagger: + +* `#Plan`: a complete configuration executable by `dagger up` +* `#FS` to reference filesystem state +* `#Secret` to (securely) reference external secrets +* `#Service` to reference network service endpoints +* `#Stream` to reference byte streams + +### Low-level engine API + +*Import path: [`dagger.io/dagger/engine`](./stdlib/dagger/engine)* + +`engine` is a low-level API for accessing the raw capabilities of the Dagger Engine. Most developers should use the Dagger Core API instead (`dagger.io/dagger`), but experts and framework developers can target the engine API directly for maximum control. + +This API prioritizes robustness, consistency, and feature completeness. It does NOT prioritize developer convenience or leveraging Cue for composition. + +In Europa, `engine` will deprecate the following implicit API: +* Low-level operations defined in `alpha.dagger.io/dagger/op` +* Imperative DSL to assemble Dockerfile-like arrays as Cue arrays +* Convention to embed pipelines in the Cue lattice with the special nested definition `#up` +* Convention to reference filesystem state from the Cue lattice with `@dagger(artifact)` +* Convention to reference external secrets from the Cue lattice with `@dagger(secret)` +* Convention to reference external network endpoints from the Cue lattive with `@dagger(stream)` +* Convention that some operations (specifically `op.#Local`) are meant to be generated at runtime rather than authored manually. + +### Docker API + +*Import path: [`universe.dagger.io/docker`](./universe/docker)* + +The `docker` package is a native Cue API for Docker. You can use it to build, run, push and pull Docker containers directly from Cue. + +The Dagger container API defines the following types: + +* `#Image`: a container image +* `#Run`: run a comand in a container +* `#Push`: upload an image to a repository +* `#Pull`: download an image from a repository +* `#Build`: build an image + +### Examples + +*Import path: [`universe.dagger.io/examples`](https://github.com/shykes/dagger/tree/llb2/europa/universe/examples)* + +This package contains examples of complete Dagger configurations, including the result of following tutorials in the documentations. + +For example, [the todoapp example](https://github.com/shykes/dagger/tree/llb2/europa/universe/examples/todoapp/deploy) corresponds to the [Getting Started tutorial](https://docs.dagger.io/1003/get-started/) + +### More packages + +More packages are being developed under [universe.dagger.io](./universe) + + +## TODO LIST + +* Support native language dev in `docker.#Run` with good DX (Python, Go, Typescript etc.) +* #Scratch: replace with null #FS? +* Coding style. When to use verbs vs. nouns? +* Resolve registry auth special case (buildkit does not support scoping registry auth) +* Easy file injection API (`container.#Image.files` ?) +* Use file injection instead of inline for `#Command.script` (to avoid hitting arg character limits) +* Organize universe packages in sub-categories? +* Are there runtime limitations in…. + * using hidden fields `_foo` as part of the DAG? + * using `if` statements as part of the DAG? + * using inlined Cue expressions as part of the DAG? +* Do we really need CUE definitions? cue/cmd doesn’t need them… This one is 💣🔥, pure speculation. We must pursue the best DX wherever that may lead us! +* Readability of error messages + * At a minimum don’t make it worse! + * Small improvements are good (eg. +* Make sure we don’t make error messages LESS readable +* Add input.params as proposed by Richard +* Combining all container operations under an opinionated universe.dagger.io/docker package: [good or bad idea](https://github.com/dagger/dagger/pull/1117#discussion_r765178454)? +* [Outstanding questions on proxy features](https://github.com/dagger/dagger/pull/1117#discussion_r765211280) +* [Outstanding questions on #Stream and emulating unix pipes with them](https://github.com/dagger/dagger/pull/1117#discussion_r766145864) +* [Outstanding questions on engine.#Pull and information loss](https://github.com/dagger/dagger/pull/1117#discussion_r765219049) +* [Outstanding questions on global registry auth scope in buildkit](https://github.com/dagger/dagger/pull/1117#discussion_r765963051) +* [Outstanding questions on platform key](https://github.com/dagger/dagger/pull/1117#discussion_r766085610) diff --git a/europa/fmt.sh b/europa/fmt.sh new file mode 100755 index 00000000..b2121b3f --- /dev/null +++ b/europa/fmt.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +find . -name '*.cue' -exec cue fmt -s {} \; diff --git a/europa/search.sh b/europa/search.sh new file mode 100755 index 00000000..a6496286 --- /dev/null +++ b/europa/search.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +find . -name '*.cue' -exec grep -H "$1" {} \; diff --git a/europa/stdlib/.gitignore b/europa/stdlib/.gitignore new file mode 100644 index 00000000..ff8f4176 --- /dev/null +++ b/europa/stdlib/.gitignore @@ -0,0 +1,2 @@ +node_modules +report.xml diff --git a/europa/stdlib/cue.mod/module.cue b/europa/stdlib/cue.mod/module.cue new file mode 100644 index 00000000..b2757079 --- /dev/null +++ b/europa/stdlib/cue.mod/module.cue @@ -0,0 +1 @@ +module: "dagger.io" diff --git a/europa/stdlib/dagger/engine/exec.cue b/europa/stdlib/dagger/engine/exec.cue new file mode 100644 index 00000000..83a1c792 --- /dev/null +++ b/europa/stdlib/dagger/engine/exec.cue @@ -0,0 +1,64 @@ +package engine + +// Execute a command in a container +#Exec: { + _exec: {} + + // Container filesystem + input: #FS + + // Mounts + mounts: [...#Mount] + + // Command to execute + args: [...string] | string + + // Environment variables + environ: [...string] + + // Working directory + workdir?: string + + // Optionally attach to command standard input stream + stdin?: #Stream + + // Optionally attach to command standard output stream + stdout?: #Stream + + // Optionally attach to command standard error stream + stderr?: #Stream + + // Modified filesystem + output: #FS + + // Command exit code + exit: int +} + +// A transient filesystem mount. +#Mount: { + dest: string + { + contents: #CacheDir | #TempDir | #Service + } | { + contents: #FS + source: string | *"/" + ro: true | *false + } | { + contents: #Secret + uid: uint32 | *0 + gid: uint32 | *0 + optional: true | *false + } +} + +// A (best effort) persistent cache dir +#CacheDir: { + id: string + concurrency: *"shared" | "private" | "locked" +} + +// A temporary directory for command execution +#TempDir: { + size?: int64 +} diff --git a/europa/stdlib/dagger/engine/fs.cue b/europa/stdlib/dagger/engine/fs.cue new file mode 100644 index 00000000..29b04e4c --- /dev/null +++ b/europa/stdlib/dagger/engine/fs.cue @@ -0,0 +1,56 @@ +package engine + +// A filesystem state +#FS: { + _fs: ID: string +} + +// Produce an empty directory +// FIXME: replace with a null value for #FS? +#Scratch: { + _scratch: {} + + output: #FS +} + +#ReadFile: { + _readFile: {} + + input: #FS + path: string + contents: string + output: #FS +} + +#WriteFile: { + _writeFile: {} + + input: #FS + path: string + contents: string + output: #FS +} + +#Copy: { + _copy: {} + + input: #FS + #CopyInfo + output: #FS +} + +#CopyInfo: { + source: { + root: #FS + path: string | *"/" + } + dest: string +} + +#Merge: { + _merge: {} + + input: #FS + layers: [...#CopyInfo] + output: #FS +} diff --git a/europa/stdlib/dagger/engine/git.cue b/europa/stdlib/dagger/engine/git.cue new file mode 100644 index 00000000..6a0e95c8 --- /dev/null +++ b/europa/stdlib/dagger/engine/git.cue @@ -0,0 +1,19 @@ +package engine + +// Push a directory to a git remote +#GitPush: { + gitPush: {} + + input: #FS + remote: string + ref: string +} + +// Pull a directory from a git remote +#GitPull: { + gitPull: {} + + remote: string + ref: string + output: #FS +} diff --git a/europa/stdlib/dagger/engine/image.cue b/europa/stdlib/dagger/engine/image.cue new file mode 100644 index 00000000..3acfeff0 --- /dev/null +++ b/europa/stdlib/dagger/engine/image.cue @@ -0,0 +1,88 @@ +package engine + +// Container image config +// See https://opencontainers.org +#ImageConfig: { + env?: [...string] + user?: string + command?: [...string] + // FIXME +} + +// Upload a container image to a remote repository +#Push: { + push: {} + + // Target repository address + dest: #Ref + + // Filesystem contents to push + input: #FS + + // Container image config + config: #ImageConfig + + // Authentication + auth: [...{ + target: string + username: string + secret: string | #Secret + }] + + // Complete ref of the pushed image, including digest + result: #Ref +} + +// Download a container image from a remote repository +#Pull: { + pull: {} + + // Repository source ref + source: #Ref + // Authentication + auth: [...{ + target: string + username: string + secret: string | #Secret + }] + + // Root filesystem of downloaded image + output: #FS + + // Complete ref of downloaded image (including digest) + result: #Ref + + // Downloaded container image config + config: #ImageConfig +} + +// A ref is an address for a remote container image +// +// Examples: +// - "index.docker.io/dagger" +// - "dagger" +// - "index.docker.io/dagger:latest" +// - "index.docker.io/dagger:latest@sha256:a89cb097693dd354de598d279c304a1c73ee550fbfff6d9ee515568e0c749cfe" +#Ref: string + +// Build a container image using buildkit +#Build: { + build: {} + + // Source directory to build + source: #FS + { + frontend: "dockerfile" + dockerfile: { + path: string | *"Dockerfile" + } | { + contents: string + } + } + + // Root filesystem produced by build + output: #FS + + // Container image config produced by build + config: #ImageConfig +} diff --git a/europa/stdlib/dagger/engine/secret.cue b/europa/stdlib/dagger/engine/secret.cue new file mode 100644 index 00000000..c6bd8e04 --- /dev/null +++ b/europa/stdlib/dagger/engine/secret.cue @@ -0,0 +1,6 @@ +package engine + +// An external secret +#Secret: { + _secret: ID: string +} diff --git a/europa/stdlib/dagger/engine/service.cue b/europa/stdlib/dagger/engine/service.cue new file mode 100644 index 00000000..aede2d71 --- /dev/null +++ b/europa/stdlib/dagger/engine/service.cue @@ -0,0 +1,6 @@ +package engine + +// An external network service +#Service: { + _service: ID: string +} diff --git a/europa/stdlib/dagger/engine/stream.cue b/europa/stdlib/dagger/engine/stream.cue new file mode 100644 index 00000000..865ded97 --- /dev/null +++ b/europa/stdlib/dagger/engine/stream.cue @@ -0,0 +1,6 @@ +package engine + +// A stream of bytes +#Stream: { + _stream: ID: string +} diff --git a/europa/stdlib/dagger/plan.cue b/europa/stdlib/dagger/plan.cue new file mode 100644 index 00000000..0bc2489e --- /dev/null +++ b/europa/stdlib/dagger/plan.cue @@ -0,0 +1,97 @@ +// The Dagger API. +package dagger + +// A deployment plan executed by `dagger up` +#Plan: #DAG + +// A special kind of program which `dagger` can execute. +#DAG: { + // Receive inputs from the client + input: { + directories: [name=string]: #InputDirectory + secrets: [name=string]: #InputSecret + } + + // Send outputs to the client + output: { + directories: [name=string]: #OutputDirectory + } + + // Forward network services to and from the client + proxy: [name=string]: #ProxyEndpoint + + // Execute actions in containers + actions: { + ... + } +} + +#InputDirectory: { + // Import from this path ON THE CLIENT MACHINE + // Example: "/Users/Alice/dev/todoapp/src" + source: string + + // Filename patterns to include + // Example: ["*.go", "Dockerfile"] + include?: [...string] + + // Filename patterns to exclude + // Example: ["node_modules"] + exclude?: [...string] + + // Imported filesystem contents + // Use this as input for actions requiring an #FS field + contents: #FS +} + +#OutputDirectory: { + // Filesystem contents to export + // Reference an #FS field produced by an action + contents: #FS + + // Export to this path ON THE CLIENT MACHINE + dest: string +} + +// Securely receive a secret from the client +#InputSecret: { + // Reference to the secret contents + // Use this by securely mounting it into a container. + // See universe.dagger.io/docker.#Run.mounts + contents: #Secret + + { + // Execute a command ON THE CLIENT MACHINE and read secret from standard output + command: [string, ...string] | string + // Execute command in an interactive terminal + // for example to prompt for a passphrase + interactive: true | *false + } | { + // Read secret from a file ON THE CLIENT MACHINE + path: string + } | { + // Read secret from an environment variable ON THE CLIENT MACHINE + envvar: string + } +} + +// Forward a network endpoint to and from the client +#ProxyEndpoint: { + // Service endpoint can be proxied to action containers as unix sockets + // FIXME: should #Service be renamed to #ServiceEndpoint or #Endpoint? Naming things is hard... + endpoint: #Service + + { + // Listen for connections ON THE CLIENT MACHINE, proxy to actions + listen: #Address + } | { + // Connect to a remote endpoint FROM THE CLIENT MACHINE, proxy to actions + connect: #Address + } | { + // Proxy to/from the contents of a file ON THE CLIENT MACHINE + filepath: string + } | { + // Proxy to/from standard input and output of a command ON THE CLIENT MACHINE + command: [string, ...string] | string + } +} diff --git a/europa/stdlib/dagger/types.cue b/europa/stdlib/dagger/types.cue new file mode 100644 index 00000000..c6b95779 --- /dev/null +++ b/europa/stdlib/dagger/types.cue @@ -0,0 +1,36 @@ +package dagger + +import ( + "dagger.io/dagger/engine" +) + +// A reference to a filesystem tree. +// For example: +// - The root filesystem of a container +// - A source code repository +// - A directory containing binary artifacts +// Rule of thumb: if it fits in a tar archive, it fits in a #FS. +#FS: engine.#FS + +// A reference to an external secret, for example: +// - A password +// - A SSH private key +// - An API token +// Secrets are never merged in the Cue tree. They can only be used +// by a special filesystem mount designed to minimize leak risk. +#Secret: engine.#Secret + +// A reference to a stream of bytes, for example: +// - The standard output or error stream of a command +// - The standard input stream of a command +// - The contents of a file or named pipe +#Stream: engine.#Stream + +// A reference to a network service endpoint, for example: +// - A TCP or UDP port +// - A unix socket +// - An HTTPS endpoint +#Service: engine.#Service + +// A network service address +#Address: string & =~"^(tcp://|unix://|udp://).*" diff --git a/europa/stdlib/dagger/utils.cue b/europa/stdlib/dagger/utils.cue new file mode 100644 index 00000000..e90366e6 --- /dev/null +++ b/europa/stdlib/dagger/utils.cue @@ -0,0 +1,26 @@ +package dagger + +import ( + "dagger.io/dagger/engine" +) + +// Select a subdirectory from a filesystem tree +#Subdir: { + // Input tree + input: #FS + + // Path of the subdirectory + // Example: "/build" + path: string + + // Subdirectory tree + output: #FS & _copy.output + + _copy: engine.#Copy & { + "input": engine.#Scratch.output + source: { + root: input + "path": path + } + } +} diff --git a/europa/test.sh b/europa/test.sh new file mode 100755 index 00000000..42f1510a --- /dev/null +++ b/europa/test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# Run all tests from universe/ +cd universe + +targets=( + dagger.io/dagger + dagger.io/dagger/engine + + ./docker + ./docker/test/build + + ./alpine + ./alpine/tests/simple + + ./yarn + ./yarn/tests/simple + + ./bash + ./python + ./git + ./nginx + ./netlify + ./netlify/test/simple + + ./examples/todoapp + ./examples/todoapp/dev + ./examples/todoapp/staging +) + +for t in "${targets[@]}"; do + echo "-- $t" + cue eval "$t" >/dev/null +done diff --git a/europa/universe/alpine/alpine.cue b/europa/universe/alpine/alpine.cue new file mode 100644 index 00000000..f50fab9b --- /dev/null +++ b/europa/universe/alpine/alpine.cue @@ -0,0 +1,36 @@ +// Base package for Alpine Linux +package alpine + +import ( + "universe.dagger.io/docker" +) + +// Default Alpine version +let defaultVersion = "3.13.5@sha256:69e70a79f2d41ab5d637de98c1e0b055206ba40a8145e7bddb55ccc04e13cf8f" + +// Build an Alpine Linux container image +#Build: { + // Alpine version to install + version: string | *defaultVersion + + // List of packages to install + packages: [pkgName=string]: version: string | *"" + + docker.#Build & { + steps: [ + docker.#Pull & { + source: "index.docker.io/alpine:\(version)" + }, + for pkgName, pkg in packages { + run: cmd: { + name: "apk" + args: ["add", "\(pkgName)\(version)"] + flags: { + "-U": true + "--no-cache": true + } + } + }, + ] + } +} diff --git a/europa/universe/alpine/tests/simple/simple.cue b/europa/universe/alpine/tests/simple/simple.cue new file mode 100644 index 00000000..5c4f047f --- /dev/null +++ b/europa/universe/alpine/tests/simple/simple.cue @@ -0,0 +1,38 @@ +package alpine + +import ( + "universe.dagger.io/docker" +) + +TestImageVersion: { + build: #Build & { + // install an old version on purpose + version: "3.10.9" + } + + check: docker.#Run & { + image: build.output + output: files: "/etc/alpine-release": contents: "3.10.9" + } +} + +TestPackageInstall: { + build: #Build & { + packages: { + jq: {} + curl: {} + } + } + + check: docker.#Run & { + script: """ + jq --version > /jq-version.txt + curl --version > /curl-version.txt + """ + + output: files: { + "/jq-version.txt": contents: "FIXME" + "/curl-version.txt": contents: "FIXME" + } + } +} diff --git a/europa/universe/bash/bash.cue b/europa/universe/bash/bash.cue new file mode 100644 index 00000000..28318151 --- /dev/null +++ b/europa/universe/bash/bash.cue @@ -0,0 +1,21 @@ +// Helpers to run bash commands in containers +package bash + +import ( + "universe.dagger.io/docker" +) + +// Run a bash command or script in a container +#Run: docker.#Run & { + script: string + cmd: { + name: "bash" + flags: { + "-c": script + "--noprofile": true + "--norc": true + "-e": true + "-o": "pipefail" + } + } +} diff --git a/europa/universe/cue.mod/module.cue b/europa/universe/cue.mod/module.cue new file mode 100644 index 00000000..782680c6 --- /dev/null +++ b/europa/universe/cue.mod/module.cue @@ -0,0 +1 @@ +module: "universe.dagger.io" diff --git a/europa/universe/cue.mod/pkg/.gitignore b/europa/universe/cue.mod/pkg/.gitignore new file mode 100644 index 00000000..a572e9ee --- /dev/null +++ b/europa/universe/cue.mod/pkg/.gitignore @@ -0,0 +1,2 @@ +# dagger universe +alpha.dagger.io diff --git a/europa/universe/cue.mod/pkg/dagger.io b/europa/universe/cue.mod/pkg/dagger.io new file mode 120000 index 00000000..961afc64 --- /dev/null +++ b/europa/universe/cue.mod/pkg/dagger.io @@ -0,0 +1 @@ +../../../stdlib \ No newline at end of file diff --git a/europa/universe/docker/build.cue b/europa/universe/docker/build.cue new file mode 100644 index 00000000..c7ec2128 --- /dev/null +++ b/europa/universe/docker/build.cue @@ -0,0 +1,97 @@ +package docker + +import ( + "dagger.io/dagger" + "dagger.io/dagger/engine" +) + +// Modular build API for Docker containers +#Build: { + steps: [#Step, ...#Step] + output: #Image + + // Generate build DAG from linerar steps + dag: { + for idx, step in steps { + // As a special case, wrap #Run into a valid step + if step.run != _|_ { + "\(idx)": { + input: _ + run: step & { + image: input + output: rootfs: _ + } + output: { + config: input.config + rootfs: run.output.rootfs + } + } + } + + // Otherwise, just use the step as is + if step.run == _|_ { + "\(idx)": { + run: false + step + } + } + + // Either way, connect input to previous output + if idx > 0 { + "\(idx)": input: dag["\(idx-1)"].output + } + } + } + + if len(dag) > 0 { + output: dag["\(len(dag)-1)"].output + } +} + +// A build step is anything that produces a docker image +#Step: { + input?: #Image + output: #Image + ... +} | #Run + +// Build step that copies files into the container image +#Copy: { + input: #Image + contents: dagger.#FS + source: string | *"/" + dest: string | *"/" + + // Execute copy operation + copy: engine.#Copy & { + "input": input.rootfs + "source": { + root: contents + path: source + } + dest: copy.dest + } + + output: #Image & { + config: input.config + rootfs: copy.output + } +} + +// Build step that executes a Dockerfile +#Dockerfile: { + // Source directory + source: dagger.#FS + + // FIXME: not yet implemented + *{ + // Look for Dockerfile in source at default path + path: "Dockerfile" + } | { + // Look for Dockerfile in source at a custom path + path: string + } | { + // Custom dockerfile contents + contents: string + } +} diff --git a/europa/universe/docker/docker.cue b/europa/universe/docker/docker.cue new file mode 100644 index 00000000..9f88d192 --- /dev/null +++ b/europa/universe/docker/docker.cue @@ -0,0 +1,205 @@ +// Build, ship and run Docker containers in Dagger +package docker + +import ( + "list" + + "dagger.io/dagger/engine" + "dagger.io/dagger" +) + +// A container image +#Image: { + // Root filesystem of the image. + rootfs: dagger.#FS + + // Image config + config: engine.#ImageConfig +} + +// Run a command in a container +#Run: { + run: true // FIXME + image: #Image + + always: bool | *false + + // Filesystem mounts + mounts: [name=string]: engine.#Mount + + // Expose network ports + ports: [name=string]: { + frontend: dagger.#Service + backend: { + protocol: *"tcp" | "udp" + address: string + } + } + + // Command to execute + cmd: { + // Name of the command to execute + // Examples: "ls", "/bin/bash" + name: string + + // Positional arguments to the command + // Examples: ["/tmp"] + args: [...string] + + // Command-line flags represented in a civilized form + // Example: {"-l": true, "-c": "echo hello world"} + flags: [string]: (string | true) + + _flatFlags: list.FlattenN([ + for k, v in flags { + if (v & bool) != _|_ { + [k] + } + if (v & string) != _|_ { + [k, v] + } + }, + ], 1) + } + + // Optionally pass a script to interpret + // Example: "echo hello\necho world" + script?: string + if script != _|_ { + // Default interpreter is /bin/sh -c + cmd: *{ + name: "/bin/sh" + flags: "-c": script + } | {} + } + + // Environment variables + // Example: {"DEBUG": "1"} + env: [string]: string + + // Working directory for the command + // Example: "/src" + workdir: string | *"/" + + // Username or UID to ad + // User identity for this command + // Examples: "root", "0", "1002" + user: string + + // Optionally attach to command standard streams + stdin: dagger.#Stream | *null + stdout: dagger.#Stream | *null + stderr: dagger.#Stream | *null + + // Output fields + { + // Has the command completed? + completed: bool & (_exec.exit != _|_) + + // Was completion successful? + success: bool & (_exec.exit == 0) + + // Details on error, if any + error: { + // Error code + code: _exec.exit + + // Error message + message: string | *null + } + + output: { + rootfs?: dagger.#FS & _exec.output + files: [path=string]: { + contents: string + contents: _read.contents + + _read: engine.#ReadFile & { + input: _exec.output + "path": path + } + } + directories: [path=string]: { + contents: dagger.#FS + contents: (dagger.#Subdir & { + input: _exec.output + "path": path + }).output + } + } + } + + // Actually execute the command + _exec: engine.#Exec & { + args: [cmd.name] + cmd._flatFlags + cmd.args + input: image.rootfs + "mounts": [ for mnt in mounts {mnt}] + environ: [ for k, v in env {"\(k)=\(v)"}] + "workdir": workdir + "stdin": stdin + // FIXME: user + } +} + +// A ref is an address for a remote container image +// Examples: +// - "index.docker.io/dagger" +// - "dagger" +// - "index.docker.io/dagger:latest" +// - "index.docker.io/dagger:latest@sha256:a89cb097693dd354de598d279c304a1c73ee550fbfff6d9ee515568e0c749cfe" +#Ref: engine.#Ref + +// Download an image from a remote registry +#Pull: { + // Source ref. + source: #Ref + + // Registry authentication + // Key must be registry address, for example "index.docker.io" + auth: [registry=string]: { + username: string + secret: dagger.#Secret + } + + _op: engine.#Pull & { + "source": source + "auth": [ for target, creds in auth { + "target": target + creds + }] + } + + // Downloaded image + image: #Image & { + rootfs: _op.output + config: _op.config + } + + // FIXME: compat with Build API + output: image +} + +// Upload an image to a remote repository +#Push: { + // Destination ref + dest: #Ref + + // Complete ref after pushing (including digest) + result: #Ref & _push.result + + // Registry authentication + // Key must be registry address + auth: [registry=string]: { + username: string + secret: dagger.#Secret + } + + // Image to push + image: #Image + + _push: engine.#Push & { + dest: dest + input: image.rootfs + config: image.config + } +} diff --git a/europa/universe/docker/test/bash/bash.cue b/europa/universe/docker/test/bash/bash.cue new file mode 100644 index 00000000..20c458e7 --- /dev/null +++ b/europa/universe/docker/test/bash/bash.cue @@ -0,0 +1,15 @@ +package docker + +cmd: #Command & { + script: "echo hello world" + exec: { + name: "/bin/bash" + flags: { + "-c": script + "--noprofile": true + "--norc": true + "-e": true + "-o": "pipefail" + } + } +} diff --git a/europa/universe/docker/test/build/build.cue b/europa/universe/docker/test/build/build.cue new file mode 100644 index 00000000..ae6d6f92 --- /dev/null +++ b/europa/universe/docker/test/build/build.cue @@ -0,0 +1,82 @@ +package docker + +import ( + "dagger.io/dagger" + "universe.dagger.io/nginx" +) + +build0: #Build & { + steps: [ + { + output: #Image & { + config: user: "foo" + } + }, + ] +} + +// Inventory of real-world build use cases +// +// 1. Build netlify image +// - import alpine base +// - execute 'yarn add netlify' + +// 2. Build todoapp dev image +// - import nginx base +// - copy app directory into /usr/share/nginx/html + +build2: { + source: dagger.#FS + + build: #Build & { + steps: [ + nginx.#Build & { + flavor: "alpine" + }, + { + // Custom step to watermark the image + input: _ + output: input & { + config: user: "42" + } + }, + #Copy & { + contents: source + dest: "/usr/share/nginx/html" + }, + ] + } + img: build.output + + // Assert: + img: config: user: "42" +} + +// 3. Build alpine base image +// - pull from docker hub +// - execute 'apk add' once per package + +// 4. Build app from dockerfile + +// 5. execute several commands in a row + +build3: { + build: #Build & { + steps: [ + #Pull & { + source: "alpine" + }, + #Run & { + script: "echo A > /A.txt" + }, + #Run & { + script: "echo B > /B.txt" + }, + #Run & { + script: "echo C > /C.txt" + }, + ] + } + + result: build.output +} diff --git a/europa/universe/examples/README.md b/europa/universe/examples/README.md new file mode 100644 index 00000000..d80d0452 --- /dev/null +++ b/europa/universe/examples/README.md @@ -0,0 +1,3 @@ +## Dagger examples + +A collection of examples to help Dagger developers get started. diff --git a/europa/universe/examples/todoapp/base.cue b/europa/universe/examples/todoapp/base.cue new file mode 100644 index 00000000..bb89f149 --- /dev/null +++ b/europa/universe/examples/todoapp/base.cue @@ -0,0 +1,28 @@ +// Deployment plan for Dagger's example todoapp +package todoapp + +import ( + "dagger.io/dagger" + + "universe.dagger.io/git" + "universe.dagger.io/yarn" +) + +dagger.#DAG & { + // Build the app with yarn + actions: build: yarn.#Build + + // Wire up source code to build + { + input: directories: source: _ + actions: build: source: input.directories.source.contents + } | { + actions: { + pull: git.#Pull & { + remote: "https://github.com/mdn/todo-react" + ref: "master" + } + build: source: pull.checkout + } + } +} diff --git a/europa/universe/examples/todoapp/dev/dev.cue b/europa/universe/examples/todoapp/dev/dev.cue new file mode 100644 index 00000000..935999b7 --- /dev/null +++ b/europa/universe/examples/todoapp/dev/dev.cue @@ -0,0 +1,40 @@ +// Local dev environment for todoapp +package todoapp + +import ( + "universe.dagger.io/docker" + "universe.dagger.io/nginx" +) + +// Expose todoapp web port +proxy: web: _ + +actions: { + // Reference app build inherited from base config + build: _ + _app: build.output + + container: { + // Build a container image serving the app with nginx + build: docker.#Build & { + steps: [ + nginx.#Build & { + flavor: "alpine" + }, + docker.#Copy & { + contents: _app + dest: "/usr/share/nginx/html" + }, + ] + } + + // Run the app in an ephemeral container + run: docker.#Run & { + image: build.output + ports: web: { + frontend: proxy.web.endpoint + backend: address: "localhost:5000" + } + } + } +} diff --git a/europa/universe/examples/todoapp/staging/staging.cue b/europa/universe/examples/todoapp/staging/staging.cue new file mode 100644 index 00000000..8125ffcf --- /dev/null +++ b/europa/universe/examples/todoapp/staging/staging.cue @@ -0,0 +1,25 @@ +// Deploy to Netlify +package todoapp + +import ( + "universe.dagger.io/netlify" +) + +// Netlify API token +input: secrets: netlify: _ + +// Must be a valid branch/PR name +environment: string + +actions: { + + // Yarn build inherited from base config + build: _ + + deploy: netlify.#Deploy & { + contents: build.output + token: input.secrets.netlify.contents + site: *"acme-inc-\(environment)" | string + team: *"acme-inc" | string + } +} diff --git a/europa/universe/git/git.cue b/europa/universe/git/git.cue new file mode 100644 index 00000000..9130b676 --- /dev/null +++ b/europa/universe/git/git.cue @@ -0,0 +1,8 @@ +package git + +import ( + "dagger.io/dagger" +) + +#Pull: dagger.#GitPull +#Push: dagger.#GitPush diff --git a/europa/universe/netlify/deploy.sh b/europa/universe/netlify/deploy.sh new file mode 120000 index 00000000..6a5d73ae --- /dev/null +++ b/europa/universe/netlify/deploy.sh @@ -0,0 +1 @@ +deploy.sh.cue \ No newline at end of file diff --git a/europa/universe/netlify/deploy.sh.cue b/europa/universe/netlify/deploy.sh.cue new file mode 100644 index 00000000..019dd491 --- /dev/null +++ b/europa/universe/netlify/deploy.sh.cue @@ -0,0 +1,56 @@ +package netlify + +_deployScript: #""" + export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)" + + create_site() { + url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" + + response=$(curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + $url \ + -d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body + ) + if [ $? -ne 0 ]; then + cat body >&2 + exit 1 + fi + + cat body | jq -r '.site_id' + } + + site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + https://api.netlify.com/api/v1/sites\?filter\=all | \ + jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \ + ) + if [ -z "$site_id" ] ; then + if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then + echo "Site $NETLIFY_SITE_NAME does not exist" + exit 1 + fi + site_id=$(create_site) + if [ -z "$site_id" ]; then + echo "create site failed" + exit 1 + fi + fi + + netlify link --id "$site_id" + netlify build + + netlify deploy \ + --dir="$(pwd)" \ + --site="$site_id" \ + --prod \ + | tee /tmp/stdout + + url=$( /netlify/url + printf "$deployUrl" > /netlify/deployUrl + printf "$logsUrl" > /netlify/logsUrl + """# diff --git a/europa/universe/netlify/netlify.cue b/europa/universe/netlify/netlify.cue new file mode 100644 index 00000000..8c94079d --- /dev/null +++ b/europa/universe/netlify/netlify.cue @@ -0,0 +1,99 @@ +// Deploy to Netlify +// https://netlify.com +package netlify + +import ( + "dagger.io/dagger" + "universe.dagger.io/docker" + + "universe.dagger.io/alpine" + "universe.dagger.io/bash" +) + +// Deploy a site to Netlify +#Deploy: { + // Contents of the site + contents: dagger.#FS + + // Name of the Netlify site + // Example: "my-super-site" + site: string + + // Netlify API token + token: dagger.#Secret + + // Name of the Netlify team (optional) + // Example: "acme-inc" + // Default: use the Netlify account's default team + team: string | *"" + + // Domain at which the site should be available (optional) + // If not set, Netlify will allocate one under netlify.app. + // Example: "www.mysupersite.tld" + domain: string | *null + + // Create the site if it doesn't exist + create: *true | false + + // Execute `netlify deploy` in a container + command: bash.#Run & { + // Container image. `netlify` must be available in the execution path + *{ + _buildDefaultImage: docker.#Build & { + input: alpine.#Build & { + bash: version: "=~5.1" + jq: version: "=~1.6" + curl: {} + yarn: version: "=~1.22" + } + steps: [{ + run: script: "yarn global add netlify-cli@3.38.10" + }] + } + + // No nested tasks, boo hoo hoo + image: _buildDefaultImage.output + env: CUSTOM_IMAGE: "0" + } | { + env: CUSTOM_IMAGE: "1" + } + + script: _deployScript // see deploy.sh + always: true + env: { + NETLIFY_SITE_NAME: site + if (create) { + NETLIFY_SITE_CREATE: "1" + } + if domain != null { + NETLIFY_DOMAIN: domain + } + NETLIFY_ACCOUNT: team + } + workdir: "/src" + mounts: { + "Site contents": { + dest: "/src" + "contents": contents + } + "Netlify token": { + dest: "/run/secrets/token" + contents: token + } + } + output: files: { + "/netlify/url": _ + "/netlify/deployUrl": _ + "/netlify/logsUrl": _ + } + } + + // URL of the deployed site + url: command.output.files."/netlify/url".contents + + // URL of the latest deployment + deployUrl: command.output.files."/netlify/deployUrl".contents + + // URL for logs of the latest deployment + logsUrl: command.output.files."/netlify/logsUrl".contents +} diff --git a/europa/universe/netlify/test/simple/simple.cue b/europa/universe/netlify/test/simple/simple.cue new file mode 100644 index 00000000..e9e78744 --- /dev/null +++ b/europa/universe/netlify/test/simple/simple.cue @@ -0,0 +1,9 @@ +package netlify + +import ( + "dagger.io/dagger" +) + +deploy: #Deploy & { + contents: dagger.#Scratch +} diff --git a/europa/universe/nginx/nginx.cue b/europa/universe/nginx/nginx.cue new file mode 100644 index 00000000..affadd44 --- /dev/null +++ b/europa/universe/nginx/nginx.cue @@ -0,0 +1,26 @@ +// Run and deploy the Nginx web server +// https://nginx.org +package nginx + +import ( + "universe.dagger.io/docker" +) + +// Build a nginx container image +// FIXME: bootstrapping by wrapping "docker pull nginx" +// Possible ways to improve: +// 1. "docker build" the docker hub image ourselves: https://github.com/nginxinc/docker-nginx +// 2. Reimplement same docker build in pure Cue (no more Dockerfile) +// FIXME: build from source or package distro, instead of docker pull +#Build: { + output: docker.#Image & _pull.image + + _pull: docker.#Pull + *{ + flavor: "alpine" + _pull: source: "index.docker.io/nginx:stable-alpine" + } | { + flavor: "debian" + _pull: source: "index.docker.io/nginx:stable" + } +} diff --git a/europa/universe/python/python.cue b/europa/universe/python/python.cue new file mode 100644 index 00000000..8afa2493 --- /dev/null +++ b/europa/universe/python/python.cue @@ -0,0 +1,24 @@ +// Helpers to run python programs +package python + +import ( + "universe.dagger.io/docker" + + "universe.dagger.io/alpine" +) + +// Run a python script in a container +#Run: docker.#Run & { + script: string + cmd: { + name: "python" + flags: "-c": script + } + + // As a convenience, image defaults to a ready-to-use python environment + image: docker.#Image | *_defaultImage + + _defaultImage: alpine.#Image & { + packages: python: version: "3" + } +} diff --git a/europa/universe/yarn/tests/simple/simple.cue b/europa/universe/yarn/tests/simple/simple.cue new file mode 100644 index 00000000..ce8771b8 --- /dev/null +++ b/europa/universe/yarn/tests/simple/simple.cue @@ -0,0 +1,11 @@ +package yarn + +import ( + "dagger.io/dagger/engine" +) + +b: #Build & { + source: engine.#Scratch.output +} + +out: b.output diff --git a/europa/universe/yarn/tests/testdata/package.json b/europa/universe/yarn/tests/testdata/package.json new file mode 100644 index 00000000..60c2e43d --- /dev/null +++ b/europa/universe/yarn/tests/testdata/package.json @@ -0,0 +1,11 @@ +{ + "name": "test", + "main": "index.js", + "license": { + "type": "Apache-2.0", + "url": "https://opensource.org/licenses/apache2.0.php" + }, + "scripts": { + "build": "mkdir -p ./build && echo output > ./build/test && touch .env && cp .env ./build/" + } +} diff --git a/europa/universe/yarn/tests/testdata2/package.json b/europa/universe/yarn/tests/testdata2/package.json new file mode 100644 index 00000000..94746735 --- /dev/null +++ b/europa/universe/yarn/tests/testdata2/package.json @@ -0,0 +1,12 @@ +{ + "name": "test", + "main": "index.js", + "license": { + "type": "Apache-2.0", + "url": "https://opensource.org/licenses/apache2.0.php" + }, + "scripts": { + "build": "mkdir -p ./build && cp /.env ./build/env" + } + } + \ No newline at end of file diff --git a/europa/universe/yarn/yarn.cue b/europa/universe/yarn/yarn.cue new file mode 100644 index 00000000..5f1aa879 --- /dev/null +++ b/europa/universe/yarn/yarn.cue @@ -0,0 +1,96 @@ +// Yarn is a package manager for Javascript applications +package yarn + +import ( + "strings" + + "dagger.io/dagger" + "dagger.io/dagger/engine" // FIXME: should not be needed for common cases + + "universe.dagger.io/alpine" + "universe.dagger.io/bash" +) + +// Build a Yarn package +#Build: { + // Application source code + source: dagger.#FS + + // working directory to use + cwd: *"." | string + + // Write the contents of `environment` to this file, + // in the "envfile" format + writeEnvFile: string | *"" + + // Read build output from this directory + // (path must be relative to working directory) + buildDir: string | *"build" + + // Run this yarn script + script: string | *"build" + + // Optional arguments for the script + args: [...string] | *[] + + // Secret variables + secrets: [string]: dagger.#Secret + + // Yarn version + yarnVersion: *"=~1.22" | string + + // Run yarn in a containerized build environment + command: bash.#Run & { + *{ + image: (alpine.#Build & { + bash: version: "=~5.1" + yarn: version: yarnVersion + }).image + env: CUSTOM_IMAGE: "0" + } | { + env: CUSTOM_IMAGE: "1" + } + + script: """ + # Create $ENVFILE_NAME file if set + [ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME" + + yarn --cwd "$YARN_CWD" install --production false + + opts=( $(echo $YARN_ARGS) ) + yarn --cwd "$YARN_CWD" run "$YARN_BUILD_SCRIPT" ${opts[@]} + mv "$YARN_BUILD_DIRECTORY" /build + """ + + mounts: { + "yarn cache": { + dest: "/cache/yarn" + contents: engine.#CacheDir + } + "package source": { + dest: "/src" + contents: source + } + // FIXME: mount secrets + } + + output: directories: "/build": _ + + env: { + YARN_BUILD_SCRIPT: script + YARN_ARGS: strings.Join(args, "\n") + YARN_CACHE_FOLDER: "/cache/yarn" + YARN_CWD: cwd + YARN_BUILD_DIRECTORY: buildDir + if writeEnvFile != "" { + ENVFILE_NAME: writeEnvFile + ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n") + } + } + + workdir: "/src" + } + + // The final contents of the package after build + output: command.output.directories."/build".contents +}