europa: vendor universe.dagger.io

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi
2022-01-11 18:03:08 -08:00
parent 0925ba588c
commit f3dee47353
38 changed files with 7 additions and 4 deletions

View File

@@ -16,20 +16,23 @@ import (
var (
// FS contains the filesystem of the stdlib.
//go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue dagger.io/**/*.cue dagger.io/**/*/*.cue
//go:embed */**/*.cue */**/**/*.cue
FS embed.FS
)
var (
AlphaModule = "alpha.dagger.io"
DaggerModule = "dagger.io"
EnginePackage = fmt.Sprintf("%s/dagger/engine", DaggerModule)
AlphaModule = "alpha.dagger.io"
DaggerModule = "dagger.io"
UniverseModule = "universe.dagger.io"
modules = []string{
AlphaModule,
DaggerModule,
UniverseModule,
}
EnginePackage = fmt.Sprintf("%s/dagger/engine", DaggerModule)
lockFilePath = "dagger.lock"
)

View File

@@ -0,0 +1,76 @@
# Europa Universe
## About this directory
`europa-universe/` is a staging area for the upcoming `universe.dagger.io` package namespace,
which will be shipped as part of the [Europa release](https://github.com/dagger/dagger/issues/1088).
## What is Universe?
The Dagger Universe is a catalog of reusable Cue packages, curated by Dagger but possibly authored by third parties. Most packages in Universe contain reusable actions; some may also contain entire configuration templates.
The import domain for Universe will be `universe.dagger.io`. It will deprecate the current domain `alpha.dagger.io`.
## Where is the `dagger` package?
Europa will also introduce a new package for the Dagger Core API: `dagger.io/dagger`.
This is a core package, and is *not* part of Universe (note the import domain).
The development version of the Europa core API can be imported as [alpha.dagger.io/europa/dagger](../stdlib/europa/dagger).
## Where is the `dagger/engine` package?
Europa will also introduce a new package for the Low-Level Dagger Engine API : `dagger.io/dagger/engine`.
This is a core package, and is *not* part of Universe (note the import domain).
The development version of the Europa Low-Level Engine API can be imported as either:
* [alpha.dagger.io/europa/dagger/engine/spec/engine](../stdlib/europa/dagger/engine/spec/engine) for the full spec
* [alpha.dagger.io/dagger/europa/engine](../stdlib/europa/dagger/engine) for the implemented subset of the spec
## Universe vs other packages
This table compares Dagger core packages, Dagger Universe packages, and the overall CUE package ecosystem.
| | *Dagger core* | *Dagger Universe* | *CUE ecosystem* |
|---|----------------|-------------------|-----------------|
| Import path | `dagger.io` | `universe.dagger.io` | Everything else |
| Purpose | Access core Dagger features | Safely reuse code from the Dagger community | Reuse any CUE code from anyone |
| Author | Dagger team | Dagger community, curated by Dagger | Anyone |
| Release cycle | Released with Dagger engine | Released continuously | No release cycle |
| Size | Small | Large | Very large |
| Growth rate | Grows slowly, with engine features | Grows fast, with Dagger community | Grows even faster, with CUE ecosystem |
## Notable packages
### Docker API
*Import path: [`universe.dagger.io/docker`](./universe/docker)*
The `docker` package is a native Cue API for Docker. You can use it to build, run, push and pull Docker containers directly from Cue.
The Dagger container API defines the following types:
* `#Image`: a container image
* `#Run`: run a comand in a container
* `#Push`: upload an image to a repository
* `#Pull`: download an image from a repository
* `#Build`: build an image
### Examples
*Import path: [`universe.dagger.io/examples`](./examples)*
This package contains examples of complete Dagger configurations, including the result of following tutorials in the documentations.
For example, [the todoapp example](./examples/todoapp) corresponds to the [Getting Started tutorial](https://docs.dagger.io/1003/get-started/)
## TODO LIST
* Support native language dev in `docker.#Run` with good DX (Python, Go, Typescript etc.)
* Coding style. When to use verbs vs. nouns?
* Easy file injection API (`container.#Image.files` ?)
* Use file injection instead of inline for `#Command.script` (to avoid hitting arg character limits)
* Organize universe packages in sub-categories?

View File

@@ -0,0 +1,36 @@
// Base package for Alpine Linux
package alpine
import (
"universe.dagger.io/docker"
)
// Default Alpine version
let defaultVersion = "3.13.5@sha256:69e70a79f2d41ab5d637de98c1e0b055206ba40a8145e7bddb55ccc04e13cf8f"
// Build an Alpine Linux container image
#Build: {
// Alpine version to install
version: string | *defaultVersion
// List of packages to install
packages: [pkgName=string]: version: string | *""
docker.#Build & {
steps: [
docker.#Pull & {
source: "index.docker.io/alpine:\(version)"
},
for pkgName, pkg in packages {
run: cmd: {
name: "apk"
args: ["add", "\(pkgName)\(version)"]
flags: {
"-U": true
"--no-cache": true
}
}
},
]
}
}

View File

@@ -0,0 +1,38 @@
package alpine
import (
"universe.dagger.io/docker"
)
TestImageVersion: {
build: #Build & {
// install an old version on purpose
version: "3.10.9"
}
check: docker.#Run & {
image: build.output
output: files: "/etc/alpine-release": contents: "3.10.9"
}
}
TestPackageInstall: {
build: #Build & {
packages: {
jq: {}
curl: {}
}
}
check: docker.#Run & {
script: """
jq --version > /jq-version.txt
curl --version > /curl-version.txt
"""
output: files: {
"/jq-version.txt": contents: "FIXME"
"/curl-version.txt": contents: "FIXME"
}
}
}

View File

@@ -0,0 +1,21 @@
// Helpers to run bash commands in containers
package bash
import (
"universe.dagger.io/docker"
)
// Run a bash command or script in a container
#Run: docker.#Run & {
script: string
cmd: {
name: "bash"
flags: {
"-c": script
"--noprofile": true
"--norc": true
"-e": true
"-o": "pipefail"
}
}
}

View File

@@ -0,0 +1 @@
module: "universe.dagger.io"

View File

@@ -0,0 +1 @@
../../../stdlib

View File

@@ -0,0 +1 @@
../../../stdlib/europa

View File

@@ -0,0 +1,76 @@
package docker
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
)
// Modular build API for Docker containers
#Build: {
steps: [#Step, ...#Step]
output: #Image
// Generate build DAG from linerar steps
dag: {
for idx, step in steps {
"\(idx)": step & {
// connect input to previous output
if idx > 0 {
input: dag["\(idx-1)"].output
}
}
}
}
if len(dag) > 0 {
output: dag["\(len(dag)-1)"].output
}
}
// A build step is anything that produces a docker image
#Step: {
input?: #Image
output: #Image
...
} | #Run
// Build step that copies files into the container image
#Copy: {
input: #Image
contents: dagger.#FS
source: string | *"/"
dest: string | *"/"
// Execute copy operation
copy: engine.#Copy & {
"input": input.rootfs
"source": {
root: contents
path: source
}
dest: copy.dest
}
output: #Image & {
config: input.config
rootfs: copy.output
}
}
// Build step that executes a Dockerfile
#Dockerfile: {
// Source directory
source: dagger.#FS
// FIXME: not yet implemented
*{
// Look for Dockerfile in source at default path
path: "Dockerfile"
} | {
// Look for Dockerfile in source at a custom path
path: string
} | {
// Custom dockerfile contents
contents: string
}
}

View File

@@ -0,0 +1,24 @@
package docker
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
)
// A container image
#Image: {
// Root filesystem of the image.
rootfs: dagger.#FS
// Image config
config: engine.#ImageConfig
}
// A ref is an address for a remote container image
// Examples:
// - "index.docker.io/dagger"
// - "dagger"
// - "index.docker.io/dagger:latest"
// - "index.docker.io/dagger:latest@sha256:a89cb097693dd354de598d279c304a1c73ee550fbfff6d9ee515568e0c749cfe"
// FIXME: add formatting constraints
#Ref: engine.#Ref

View File

@@ -0,0 +1,37 @@
// Build, ship and run Docker containers in Dagger
package docker
import (
"dagger.io/dagger/engine"
"dagger.io/dagger"
)
// Download an image from a remote registry
#Pull: {
// Source ref.
source: #Ref
// Registry authentication
// Key must be registry address, for example "index.docker.io"
auth: [registry=string]: {
username: string
secret: dagger.#Secret
}
_op: engine.#Pull & {
"source": source
"auth": [ for target, creds in auth {
"target": target
creds
}]
}
// Downloaded image
image: #Image & {
rootfs: _op.output
config: _op.config
}
// FIXME: compat with Build API
output: image
}

View File

@@ -0,0 +1,31 @@
package docker
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
)
// Upload an image to a remote repository
#Push: {
// Destination ref
dest: #Ref
// Complete ref after pushing (including digest)
result: #Ref & _push.result
// Registry authentication
// Key must be registry address
auth: [registry=string]: {
username: string
secret: dagger.#Secret
}
// Image to push
image: #Image
_push: engine.#Push & {
dest: dest
input: image.rootfs
config: image.config
}
}

View File

@@ -0,0 +1,130 @@
package docker
import (
"list"
"dagger.io/dagger"
"dagger.io/dagger/engine"
)
// Run a command in a container
#Run: {
image: #Image
input: image // for compatibility with #Build
always: bool | *false
// Filesystem mounts
mounts: [name=string]: engine.#Mount
// Expose network ports
// FIXME: investigate feasibility
ports: [name=string]: {
frontend: dagger.#Service
backend: {
protocol: *"tcp" | "udp"
address: string
}
}
// Command to execute
cmd: {
// Name of the command to execute
// Examples: "ls", "/bin/bash"
name: string
// Positional arguments to the command
// Examples: ["/tmp"]
args: [...string]
// Command-line flags represented in a civilized form
// Example: {"-l": true, "-c": "echo hello world"}
flags: [string]: (string | true)
_flatFlags: list.FlattenN([
for k, v in flags {
if (v & bool) != _|_ {
[k]
}
if (v & string) != _|_ {
[k, v]
}
},
], 1)
}
// Optionally pass a script to interpret
// Example: "echo hello\necho world"
script?: string
if script != _|_ {
// Default interpreter is /bin/sh -c
cmd: *{
name: "/bin/sh"
flags: "-c": script
} | {}
}
// Environment variables
// Example: {"DEBUG": "1"}
env: [string]: string
// Working directory for the command
// Example: "/src"
workdir: string | *"/"
// Username or UID to ad
// User identity for this command
// Examples: "root", "0", "1002"
user: string | *"root"
// Output fields
{
// Has the command completed?
completed: bool & (_exec.exit != _|_)
// Was completion successful?
success: bool & (_exec.exit == 0)
// Details on error, if any
error: {
// Error code
code: _exec.exit
// Error message
message: string | *null
}
output?: {
// FIXME: hack for #Build compatibility
#Image
rootfs?: dagger.#FS & _exec.output
files: [path=string]: {
contents: string
contents: _read.contents
_read: engine.#ReadFile & {
input: _exec.output
"path": path
}
}
directories: [path=string]: {
contents: dagger.#FS
contents: (dagger.#Subdir & {
input: _exec.output
"path": path
}).output
}
}
}
// Actually execute the command
_exec: engine.#Exec & {
args: [cmd.name] + cmd._flatFlags + cmd.args
input: image.rootfs
"mounts": mounts
"env": env
"workdir": workdir
"user": user
}
}

View File

@@ -0,0 +1,15 @@
package docker
cmd: #Command & {
script: "echo hello world"
exec: {
name: "/bin/bash"
flags: {
"-c": script
"--noprofile": true
"--norc": true
"-e": true
"-o": "pipefail"
}
}
}

View File

@@ -0,0 +1,65 @@
package docker
import (
"dagger.io/dagger"
"universe.dagger.io/nginx"
)
tests: {
"set config manually": build: #Build & {
steps: [{
output: #Image & {
config: user: "foo"
}
}]
}
// - import nginx base
// - copy app directory into /usr/share/nginx/html
"build static web server": {
source: dagger.#FS
build: #Build & {
steps: [
nginx.#Build & {
flavor: "alpine"
},
{
// Custom step to watermark the image
input: _
output: input & {
config: user: "42"
}
},
#Copy & {
contents: source
dest: "/usr/share/nginx/html"
},
]
}
image: build.output
// Assert:
image: config: user: "42"
}
"Run multiple commands": {
build: #Build & {
steps: [
#Pull & {
source: "alpine"
},
#Run & {
script: "echo A > /A.txt"
},
#Run & {
script: "echo B > /B.txt"
},
#Run & {
script: "echo C > /C.txt"
},
]
}
result: build.output
}
}

View File

@@ -0,0 +1,3 @@
## Dagger examples
A collection of examples to help Dagger developers get started.

View File

@@ -0,0 +1,134 @@
package ci
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/docker"
"universe.dagger.io/examples/changelog.com/highlevel/elixir/mix"
)
dagger.#Plan & {
// Receive things from client
inputs: {
directories: {
// App source code
app: _
}
secrets: {
// Docker ID password
docker: _
}
params: {
// Which Elixir base image to download
runtime_image: docker.#Ref | *"thechangelog/runtime:2021-05-29T10.17.12Z"
// Which test DB image to download
test_db_image: docker.#Ref | *"circleci/postgres:12.6"
}
}
// Do things
actions: {
// Reuse in all mix commands
_appName: "changelog"
prod: assets: docker.#Build & {
steps: [
// 1. Start from dev assets :)
dev.assets,
// 2. Mix magical command
mix.#Run & {
script: "mix phx.digest"
mix: {
env: "prod"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
workdir: _
// FIXME: remove copy-pasta
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: does this command need write access to node_modules cache?
concurrency: "readonly"
}
dest: "\(workdir)/node_modules"
}
},
]
}
dev: {
build: mix.#Build & {
env: "dev"
app: "thechangelog"
base: inputs.params.runtime_image
source: inputs.directories.app.contents
}
assets: docker.#Build & {
steps: [
// 1. Start from dev runtime build
build,
// 2. Build web assets
mix.#Run & {
mix: {
env: "dev"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
// FIXME: move this to a reusable def (yarn package? or private?)
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: will there be multiple writers?
concurrency: "locked"
}
dest: "\(workdir)/node_modules"
}
// FIXME: run 'yarn install' and 'yarn run compile' separately, with different caching?
// FIXME: can we reuse universe.dagger.io/yarn ???? 0:-)
script: "yarn install --frozen-lockfile && yarn run compile"
workdir: "/app/assets"
},
]
}
}
test: {
build: mix.#Build & {
env: "test"
app: _appName
base: inputs.params.runtime_image
source: inputs.directories.app.contents
}
// Run tests
run: docker.#Run & {
image: build.output
script: "mix test"
// Don't cache running tests
// Just because we've tested a version before, doesn't mean we don't
// want to test it again.
// FIXME: make this configurable
always: true
}
db: {
// Pull test DB image
pull: docker.#Pull & {
source: inputs.params.test_db_image
}
// Run test DB
// FIXME: kill once no longer needed (when tests are done running)
run: docker.#Run & {
image: pull.output
}
}
}
}
}

View File

@@ -0,0 +1,96 @@
package mix
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/docker"
)
// Build an Elixir application with Mix
#Build: {
// Ref to base image
// FIXME: spin out docker.#Build for max flexibility
// Perhaps implement as a custom docker.#Build step?
base: docker.#Ref
// App name (for cache scoping)
app: string
// Mix environment
env: string
// Application source code
source: dagger.#FS
docker.#Build & {
steps: [
// 1. Pull base image
docker.#Pull & {
source: base
},
// 2. Copy app source
docker.#Copy & {
contents: source
dest: "/app"
},
// 3. Download dependencies into deps cache
#Run & {
mix: {
"env": env
"app": app
depsCache: "locked"
}
workdir: "/app"
script: "mix deps.get"
},
// 4. Build!
// FIXME: step 5 is to add image data, see issue 1339
#Run & {
mix: {
"env": env
"app": app
depsCache: "private"
buildCache: "locked"
}
workdir: "/app"
script: "mix do deps.compile, compile"
},
]
}
}
// Run mix correctly in a container
#Run: {
mix: {
app: string
env: string
// FIXME: "ro" | "rw"
depsCache?: "private" | "locked"
buildCache?: "private" | "locked"
}
docker.#Run
env: MIX_ENV: mix.env
{
mix: depsCache: string
workdir: string
mounts: depsCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.depsCache
}
dest: "\(workdir)/deps"
}
} | {}
{
mix: buildCache: string
workdir: string
mounts: buildCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.buildCache
}
dest: "\(workdir)/deps"
}
} | {}
}

View File

@@ -0,0 +1,5 @@
# Low-level port of Changelog.com configuration
This is a port of the changelog.com configuration to the low-level `dagger/engine` APIs.
It currently only uses `engine` (no high-level APIs like `docker`, etc) and, because we don't currently have a way to implement long-running tasks in Europa, it doesn't parts of the config that require them, such as databases for testing. The hope is that we'll be able to figure out a way to implement that soon, and we'll update this config with those capabilities when we are able to.

View File

@@ -0,0 +1,171 @@
package main
import (
"dagger.io/dagger/engine"
)
runtime_image_ref: string | *"thechangelog/runtime:2021-05-29T10.17.12Z"
engine.#Plan & {
inputs: directories: app: {
path: "."
exclude: [
".circleci",
".dagger",
".git",
".github",
"2021",
"2022",
"_build/dev",
"_build/test",
"assets/node_modules",
"cue.mod",
"dev_docker",
"docker",
"import",
"nginx",
"priv/db",
"priv/uploads",
"script",
"tmp",
".all-contributorsrc",
".autocomplete",
".credo.exs",
".dockerignore",
".formatter.exs",
".envrc",
".env",
".gitattributes",
".gitignore",
"README.md",
"coveralls.json",
"start_dev_stack.sh",
".kube",
"erl_crash.dump",
"deps",
"_build",
"dagger",
"main.cue",
]
}
inputs: directories: docker: {
path: "."
include: [
"docker/Dockerfile.production",
".dockerignore",
]
}
actions: {
runtimeImage: engine.#Pull & {
source: runtime_image_ref
}
depsCache: engine.#CacheDir & {
id: "depsCache"
}
depsCacheMount: "depsCache": {
dest: *"/app/deps/" | string
contents: depsCache
}
buildCacheTest: engine.#CacheDir & {
id: "buildCacheTest"
}
buildCacheTestMount: "buildCacheTest": {
dest: *"/app/_build/test" | string
contents: buildCacheTest
}
buildCacheProd: engine.#CacheDir & {
id: "buildCacheProd"
}
buildCacheProdMount: "buildCacheProd": {
dest: *"/app/_build/prod" | string
contents: buildCacheProd
}
nodeModulesCache: engine.#CacheDir & {
id: "nodeModulesCache"
}
nodeModulesCacheMount: "nodeModulesCache": {
dest: *"/app/assets/node_modules" | string
contents: nodeModulesCache
}
appImage: engine.#Copy & {
input: runtimeImage.output
source: root: inputs.directories.app.contents
dest: "/app"
}
deps: engine.#Exec & {
input: appImage.output
mounts: depsCacheMount
workdir: "/app"
args: ["bash", "-c", " mix deps.get"]
}
assetsCompile: engine.#Exec & {
input: depsCompileProd.output
mounts: depsCacheMount & nodeModulesCacheMount
workdir: "/app/assets"
env: PATH: "/usr/local/lib/nodejs/node-v14.17.0-linux-x64/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
args: ["bash", "-c", "yarn install --frozen-lockfile && yarn run compile"]
}
#depsCompile: engine.#Exec & {
input: deps.output
mounts: depsCacheMount
workdir: "/app"
args: ["bash", "-c", "mix do deps.compile, compile"]
}
depsCompileTest: #depsCompile & {
env: MIX_ENV: "test"
mounts: buildCacheTestMount
}
depsCompileProd: #depsCompile & {
env: MIX_ENV: "prod"
mounts: buildCacheProdMount
}
assetsDigest: engine.#Exec & {
input: assetsCompile.output
mounts: depsCacheMount & buildCacheProdMount & nodeModulesCacheMount
env: MIX_ENV: "prod"
workdir: "/app"
args: ["bash", "-c", "mix phx.digest"]
}
imageProdCacheCopy: engine.#Exec & {
input: assetsDigest.output
mounts: (depsCacheMount & {depsCache: dest: "/mnt/app/deps/"} )
mounts: (buildCacheProdMount & {buildCacheProd: dest: "/mnt/app/_build/prod"} )
args: ["bash", "-c", "cp -Rp /mnt/app/deps/* /app/deps/ && cp -Rp /mnt/app/_build/prod/* /app/_build/prod/"]
}
imageProdDockerCopy: engine.#Copy & {
input: imageProdCacheCopy.output
source: root: inputs.directories.docker.contents
dest: "/"
}
imageProd: engine.#Build & {
source: imageProdDockerCopy.output
dockerfile: path: "/docker/Dockerfile.production"
buildArg: {
APP_FROM_PATH: "/app"
GIT_AUTHOR: "joel"
GIT_SHA: "abcdef"
APP_VERSION: "main"
BUILD_URL: "longtine.io/build"
}
}
}
}

View File

@@ -0,0 +1,250 @@
// STARTING POINT: https://docs.dagger.io/1012/ci
// + ../../../.circleci/config.yml
package main
import (
"alpha.dagger.io/dagger"
"alpha.dagger.io/docker"
"alpha.dagger.io/os"
)
app_src: dagger.#Artifact
prod_dockerfile: dagger.#Input & {string}
docker_host: dagger.#Input & {string}
dockerhub_username: dagger.#Input & {string}
dockerhub_password: dagger.#Input & {dagger.#Secret}
// ⚠️ Keep this in sync with ../docker/Dockerfile.production
runtime_image_ref: dagger.#Input & {string | *"thechangelog/runtime:2021-05-29T10.17.12Z"}
prod_image_ref: dagger.#Input & {string | *"thechangelog/changelog.com:dagger"}
build_version: dagger.#Input & {string}
git_branch: dagger.#Input & {string | *""}
git_sha: dagger.#Input & {string}
git_author: dagger.#Input & {string}
app_version: dagger.#Input & {string}
build_url: dagger.#Input & {string}
// ⚠️ Keep this in sync with manifests/changelog/db.yml
test_db_image_ref: dagger.#Input & {string | *"circleci/postgres:12.6"}
test_db_container_name: "changelog_test_postgres"
// STORY #######################################################################
//
// 1. Migrate from CircleCI to GitHub Actions
// - extract existing build pipeline into Dagger
// - run the pipeline locally
// - run the pipeline in GitHub Actions
//
// 2. Pipeline is 2x quicker (110s vs 228s)
// - optimistic branching, as pipelines were originally intended
// - use our own hardware - Linode g6-dedicated-8
// - predictable runs, no queuing
// - caching is buildkit layers
//
// 3. Open Telemetry integration out-of-the-box
// - visualise all steps in Jaeger UI
// PIPELINE OVERVIEW ###########################################################
//
// app
// |
// v
// test_db_start deps ----------------------------------------\
// | | | |
// | v v v
// | deps_compile_test deps_compile_prod assets_compile
// | | | | |
// | | | \-----|
// | v v |
// \--------------> test image_prod_cache assets_digest
// | | |
// test_db_stop <---| v |
// | image_prod <----------/
// | |
// | v
// \--------------------> image_prod_tag
//
// ========================== BEFORE | AFTER | CHANGE ===================================
// Test, build & push 370s | 10s | 37.00x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/520/workflows/fbb7c701-d25a-42c1-b42c-db514cd770b4
// + app compile 220s | 100s | 2.20x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/582/workflows/65500f3d-eccc-49da-9ab0-69846bc812a7
// + deps compile 480s | 110s | 4.36x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/532/workflows/94f5a339-52a1-45ba-b39b-1bbb69ed6488
//
// Uncached ???s | 245s | ?.??x |
//
// #############################################################################
test_db_start: docker.#Command & {
host: docker_host
env: {
CONTAINER_NAME: test_db_container_name
CONTAINER_IMAGE: test_db_image_ref
}
command: #"""
docker container inspect $CONTAINER_NAME \
--format 'Container "{{.Name}}" is "{{.State.Status}}"' \
|| docker container run \
--detach --rm --name $CONTAINER_NAME \
--publish 127.0.0.1:5432:5432 \
--env POSTGRES_USER=postgres \
--env POSTGRES_DB=changelog_test \
--env POSTGRES_PASSWORD=postgres \
$CONTAINER_IMAGE
docker container inspect $CONTAINER_NAME \
--format 'Container "{{.Name}}" is "{{.State.Status}}"'
"""#
}
app_image: docker.#Pull & {
from: runtime_image_ref
}
// Put app_src in the correct path, /app
app: os.#Container & {
image: app_image
copy: "/app": from: app_src
}
// https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#run---mounttypecache
deps_mount: "--mount=type=cache,id=deps,target=/app/deps/,sharing=shared"
build_test_mount: "--mount=type=cache,id=build_test,target=/app/_build/test,sharing=shared"
build_prod_mount: "--mount=type=cache,id=build_prod,target=/app/_build/prod,sharing=shared"
node_modules_mount: "--mount=type=cache,id=assets_node_modules,target=/app/assets/node_modules,sharing=shared"
deps: docker.#Build & {
source: app
dockerfile: """
FROM \(runtime_image_ref)
COPY /app/ /app/
WORKDIR /app
RUN \(deps_mount) mix deps.get
"""
}
assets_compile: docker.#Build & {
source: deps
dockerfile: """
FROM \(runtime_image_ref)
COPY /app/ /app/
WORKDIR /app/assets
RUN \(deps_mount) \(node_modules_mount) yarn install --frozen-lockfile && yarn run compile
"""
}
#deps_compile: docker.#Build & {
source: deps
dockerfile: """
FROM \(runtime_image_ref)
ARG MIX_ENV
ENV MIX_ENV=$MIX_ENV
COPY /app/ /app/
WORKDIR /app
RUN \(deps_mount) \(build_test_mount) \(build_prod_mount) mix do deps.compile, compile
"""
}
deps_compile_test: #deps_compile & {
args: MIX_ENV: "test"
}
test: docker.#Build & {
source: deps_compile_test
dockerfile: """
FROM \(runtime_image_ref)
ENV MIX_ENV=test
COPY /app/ /app/
WORKDIR /app
RUN \(deps_mount) \(build_test_mount) mix test
"""
}
test_db_stop: docker.#Command & {
host: docker_host
env: {
DEP: test.dockerfile
CONTAINER_NAME: test_db_container_name
}
command: #"""
docker container rm --force $CONTAINER_NAME
"""#
}
deps_compile_prod: #deps_compile & {
args: MIX_ENV: "prod"
}
assets_digest: docker.#Build & {
source: assets_compile
args: ONLY_RUN_AFTER_DEPS_COMPILE_PROD_OK: deps_compile_prod.args.MIX_ENV
dockerfile: """
FROM \(runtime_image_ref)
COPY /app/ /app/
ENV MIX_ENV=prod
WORKDIR /app/
RUN \(deps_mount) \(build_prod_mount) \(node_modules_mount) mix phx.digest
"""
}
image_prod_cache: docker.#Build & {
source: deps_compile_prod
dockerfile: """
FROM \(runtime_image_ref)
COPY /app/ /app/
WORKDIR /app
RUN --mount=type=cache,id=deps,target=/mnt/app/deps,sharing=shared cp -Rp /mnt/app/deps/* /app/deps/
RUN --mount=type=cache,id=build_prod,target=/mnt/app/_build/prod,sharing=shared cp -Rp /mnt/app/_build/prod/* /app/_build/prod/
"""
}
image_prod: docker.#Command & {
host: docker_host
copy: {
"/tmp/app": from: os.#Dir & {
from: image_prod_cache
path: "/app"
}
"/tmp/app/priv/static": from: os.#Dir & {
from: assets_digest
path: "/app/priv/static"
}
}
env: {
GIT_AUTHOR: git_author
GIT_SHA: git_sha
APP_VERSION: app_version
BUILD_VERSION: build_version
BUILD_URL: build_url
PROD_IMAGE_REF: prod_image_ref
}
files: "/tmp/Dockerfile": prod_dockerfile
secret: "/run/secrets/dockerhub_password": dockerhub_password
command: #"""
cd /tmp
docker build \
--build-arg APP_FROM_PATH=/app \
--build-arg GIT_AUTHOR="$GIT_AUTHOR" \
--build-arg GIT_SHA="$GIT_SHA" \
--build-arg APP_VERSION="$APP_VERSION" \
--build-arg BUILD_URL="$BUILD_URL" \
--tag "$PROD_IMAGE_REF" .
"""#
}
if git_branch == "master" {
image_prod_tag: docker.#Command & {
host: docker_host
env: {
DOCKERHUB_USERNAME: dockerhub_username
PROD_IMAGE_REF: image_prod.env.PROD_IMAGE_REF
ONLY_RUN_AFTER_TEST_OK: test.dockerfile
}
secret: "/run/secrets/dockerhub_password": dockerhub_password
command: #"""
docker login --username "$DOCKERHUB_USERNAME" --password "$(cat /run/secrets/dockerhub_password)"
docker push "$PROD_IMAGE_REF" | tee docker.push.log
echo "$PROD_IMAGE_REF" > image.ref
awk '/digest/ { print $3 }' < docker.push.log > image.digest
"""#
}
}

View File

@@ -0,0 +1,28 @@
// Deployment plan for Dagger's example todoapp
package todoapp
import (
"dagger.io/dagger"
"universe.dagger.io/git"
"universe.dagger.io/yarn"
)
dagger.#DAG & {
// Build the app with yarn
actions: build: yarn.#Build
// Wire up source code to build
{
input: directories: source: _
actions: build: source: input.directories.source.contents
} | {
actions: {
pull: git.#Pull & {
remote: "https://github.com/mdn/todo-react"
ref: "master"
}
build: source: pull.output
}
}
}

View File

@@ -0,0 +1,40 @@
// Local dev environment for todoapp
package todoapp
import (
"universe.dagger.io/docker"
"universe.dagger.io/nginx"
)
// Expose todoapp web port
proxy: web: _
actions: {
// Reference app build inherited from base config
build: _
_app: build.output
container: {
// Build a container image serving the app with nginx
build: docker.#Build & {
steps: [
nginx.#Build & {
flavor: "alpine"
},
docker.#Copy & {
contents: _app
dest: "/usr/share/nginx/html"
},
]
}
// Run the app in an ephemeral container
run: docker.#Run & {
image: build.output
ports: web: {
frontend: proxy.web.endpoint
backend: address: "localhost:5000"
}
}
}
}

View File

@@ -0,0 +1,25 @@
// Deploy to Netlify
package todoapp
import (
"universe.dagger.io/netlify"
)
// Netlify API token
input: secrets: netlify: _
// Must be a valid branch/PR name
environment: string
actions: {
// Yarn build inherited from base config
build: _
deploy: netlify.#Deploy & {
contents: build.output
token: input.secrets.netlify.contents
site: *"acme-inc-\(environment)" | string
team: *"acme-inc" | string
}
}

3
pkg/universe.dagger.io/fmt.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/bash
find . -name '*.cue' -exec cue fmt -s {} \;

View File

@@ -0,0 +1,8 @@
package git
import (
"dagger.io/dagger/engine"
)
#Pull: engine.#GitPull
#Push: engine.#GitPush

View File

@@ -0,0 +1 @@
deploy.sh.cue

View File

@@ -0,0 +1,56 @@
package netlify
_deployScript: #"""
export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)"
create_site() {
url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
response=$(curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-X POST -H "Content-Type: application/json" \
$url \
-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body
)
if [ $? -ne 0 ]; then
cat body >&2
exit 1
fi
cat body | jq -r '.site_id'
}
site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
https://api.netlify.com/api/v1/sites\?filter\=all | \
jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
)
if [ -z "$site_id" ] ; then
if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
echo "Site $NETLIFY_SITE_NAME does not exist"
exit 1
fi
site_id=$(create_site)
if [ -z "$site_id" ]; then
echo "create site failed"
exit 1
fi
fi
netlify link --id "$site_id"
netlify build
netlify deploy \
--dir="$(pwd)" \
--site="$site_id" \
--prod \
| tee /tmp/stdout
url=$(</tmp/stdout sed -n -e 's/^Website URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
deployUrl=$(</tmp/stdout sed -n -e 's/^Unique Deploy URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
logsUrl=$(</tmp/stdout sed -n -e 's/^Logs:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
# Write output files
mkdir -p /netlify
printf "$url" > /netlify/url
printf "$deployUrl" > /netlify/deployUrl
printf "$logsUrl" > /netlify/logsUrl
"""#

View File

@@ -0,0 +1,99 @@
// Deploy to Netlify
// https://netlify.com
package netlify
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
"universe.dagger.io/alpine"
"universe.dagger.io/bash"
)
// Deploy a site to Netlify
#Deploy: {
// Contents of the site
contents: dagger.#FS
// Name of the Netlify site
// Example: "my-super-site"
site: string
// Netlify API token
token: dagger.#Secret
// Name of the Netlify team (optional)
// Example: "acme-inc"
// Default: use the Netlify account's default team
team: string | *""
// Domain at which the site should be available (optional)
// If not set, Netlify will allocate one under netlify.app.
// Example: "www.mysupersite.tld"
domain: string | *null
// Create the site if it doesn't exist
create: *true | false
// Execute `netlify deploy` in a container
command: bash.#Run & {
// Container image. `netlify` must be available in the execution path
*{
_buildDefaultImage: docker.#Build & {
input: alpine.#Build & {
bash: version: "=~5.1"
jq: version: "=~1.6"
curl: {}
yarn: version: "=~1.22"
}
steps: [{
run: script: "yarn global add netlify-cli@3.38.10"
}]
}
// No nested tasks, boo hoo hoo
image: _buildDefaultImage.output
env: CUSTOM_IMAGE: "0"
} | {
env: CUSTOM_IMAGE: "1"
}
script: _deployScript // see deploy.sh
always: true
env: {
NETLIFY_SITE_NAME: site
if (create) {
NETLIFY_SITE_CREATE: "1"
}
if domain != null {
NETLIFY_DOMAIN: domain
}
NETLIFY_ACCOUNT: team
}
workdir: "/src"
mounts: {
"Site contents": {
dest: "/src"
"contents": contents
}
"Netlify token": {
dest: "/run/secrets/token"
contents: token
}
}
output: files: {
"/netlify/url": _
"/netlify/deployUrl": _
"/netlify/logsUrl": _
}
}
// URL of the deployed site
url: command.output.files."/netlify/url".contents
// URL of the latest deployment
deployUrl: command.output.files."/netlify/deployUrl".contents
// URL for logs of the latest deployment
logsUrl: command.output.files."/netlify/logsUrl".contents
}

View File

@@ -0,0 +1,9 @@
package netlify
import (
"dagger.io/dagger"
)
deploy: #Deploy & {
contents: dagger.#Scratch
}

View File

@@ -0,0 +1,26 @@
// Run and deploy the Nginx web server
// https://nginx.org
package nginx
import (
"universe.dagger.io/docker"
)
// Build a nginx container image
// FIXME: bootstrapping by wrapping "docker pull nginx"
// Possible ways to improve:
// 1. "docker build" the docker hub image ourselves: https://github.com/nginxinc/docker-nginx
// 2. Reimplement same docker build in pure Cue (no more Dockerfile)
// FIXME: build from source or package distro, instead of docker pull
#Build: {
output: docker.#Image & _pull.image
_pull: docker.#Pull
*{
flavor: "alpine"
_pull: source: "index.docker.io/nginx:stable-alpine"
} | {
flavor: "debian"
_pull: source: "index.docker.io/nginx:stable"
}
}

View File

@@ -0,0 +1,24 @@
// Helpers to run python programs
package python
import (
"universe.dagger.io/docker"
"universe.dagger.io/alpine"
)
// Run a python script in a container
#Run: docker.#Run & {
script: string
cmd: {
name: "python"
flags: "-c": script
}
// As a convenience, image defaults to a ready-to-use python environment
image: docker.#Image | *_defaultImage
_defaultImage: alpine.#Image & {
packages: python: version: "3"
}
}

View File

@@ -0,0 +1,3 @@
#!/bin/bash
find . -name '*.cue' -exec grep -H "$1" {} \;

33
pkg/universe.dagger.io/test.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
set -e
targets=(
dagger.io/dagger
dagger.io/dagger/engine
./docker
./docker/test/build
./alpine
./alpine/tests/simple
./yarn
./yarn/tests/simple
./bash
./python
./git
./nginx
./netlify
./netlify/test/simple
./examples/todoapp
./examples/todoapp/dev
./examples/todoapp/staging
)
for t in "${targets[@]}"; do
echo "-- $t"
cue eval "$t" >/dev/null
done

View File

@@ -0,0 +1,11 @@
package yarn
import (
"dagger.io/dagger/engine"
)
b: #Build & {
source: engine.#Scratch
}
out: b.output

View File

@@ -0,0 +1,11 @@
{
"name": "test",
"main": "index.js",
"license": {
"type": "Apache-2.0",
"url": "https://opensource.org/licenses/apache2.0.php"
},
"scripts": {
"build": "mkdir -p ./build && echo output > ./build/test && touch .env && cp .env ./build/"
}
}

View File

@@ -0,0 +1,12 @@
{
"name": "test",
"main": "index.js",
"license": {
"type": "Apache-2.0",
"url": "https://opensource.org/licenses/apache2.0.php"
},
"scripts": {
"build": "mkdir -p ./build && cp /.env ./build/env"
}
}

View File

@@ -0,0 +1,96 @@
// Yarn is a package manager for Javascript applications
package yarn
import (
"strings"
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/alpine"
"universe.dagger.io/bash"
)
// Build a Yarn package
#Build: {
// Application source code
source: dagger.#FS
// working directory to use
cwd: *"." | string
// Write the contents of `environment` to this file,
// in the "envfile" format
writeEnvFile: string | *""
// Read build output from this directory
// (path must be relative to working directory)
buildDir: string | *"build"
// Run this yarn script
script: string | *"build"
// Optional arguments for the script
args: [...string] | *[]
// Secret variables
secrets: [string]: dagger.#Secret
// Yarn version
yarnVersion: *"=~1.22" | string
// Run yarn in a containerized build environment
command: bash.#Run & {
*{
image: (alpine.#Build & {
bash: version: "=~5.1"
yarn: version: yarnVersion
}).image
env: CUSTOM_IMAGE: "0"
} | {
env: CUSTOM_IMAGE: "1"
}
script: """
# Create $ENVFILE_NAME file if set
[ -n "$ENVFILE_NAME" ] && echo "$ENVFILE" > "$ENVFILE_NAME"
yarn --cwd "$YARN_CWD" install --production false
opts=( $(echo $YARN_ARGS) )
yarn --cwd "$YARN_CWD" run "$YARN_BUILD_SCRIPT" ${opts[@]}
mv "$YARN_BUILD_DIRECTORY" /build
"""
mounts: {
"yarn cache": {
dest: "/cache/yarn"
contents: engine.#CacheDir
}
"package source": {
dest: "/src"
contents: source
}
// FIXME: mount secrets
}
output: directories: "/build": _
env: {
YARN_BUILD_SCRIPT: script
YARN_ARGS: strings.Join(args, "\n")
YARN_CACHE_FOLDER: "/cache/yarn"
YARN_CWD: cwd
YARN_BUILD_DIRECTORY: buildDir
if writeEnvFile != "" {
ENVFILE_NAME: writeEnvFile
ENVFILE: strings.Join([ for k, v in env {"\(k)=\(v)"}], "\n")
}
}
workdir: "/src"
}
// The final contents of the package after build
output: command.output.directories."/build".contents
}