kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Work in progress: re-write changelog.com example for Europa

Browse Source 
Signed-off-by: Solomon Hykes <solomon@dagger.io>
This commit is contained in:
Solomon Hykes 2021-12-15 09:10:47 +00:00 committed by Solomon Hykes
parent 6d4d10a002
commit 8b5a801380
9 changed files with 390 additions and 390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
europa-universe/docker/build.cue
View File

@ -13,28 +13,41 @@ import (
// Generate build DAG from linerar steps // Generate build DAG from linerar steps
dag: { dag: {
for idx, step in steps { for idx, step in steps {
// As a special case, wrap #Run into a valid step ///// FIXME: this section is broken and in the middle of debug + rewrite
if step.run != _|_ { //// // 1. image -> input
"\(idx)": { //// if (step.input == _|_) && ((step.image & #Image) != _|_) {
input: _ //// input: image
run: step & { //// }
image: input
output: rootfs: _
}
output: {
config: input.config
rootfs: run.output.rootfs
}
}
}
// Otherwise, just use the step as is //// // 2.
if step.run == _|_ { //// if ((step.output & docker.#Image) == _|_) && ((step.output.rootfs & dagger.#FS) != _|_) {
"\(idx)": { ////
run: false //// }
step
} //// // As a special case, wrap #Run into a valid step
} //// if step.run != _|_ {
//// "\(idx)": {
//// input: _
//// run: step & {
//// image: input
//// output: rootfs: _
//// }
//// output: {
//// config: input.config
//// rootfs: run.output.rootfs
//// }
//// }
//// }
//// // Otherwise, just use the step as is
//// if step.run == _|_ {
//// "\(idx)": {
//// run: false
//// step
//// }
//// }
"\(idx)": step
// Either way, connect input to previous output // Either way, connect input to previous output
if idx > 0 { if idx > 0 {

7
europa-universe/docker/docker.cue
View File

@ -19,8 +19,8 @@ import (
// Run a command in a container // Run a command in a container
#Run: { #Run: {
run: true // FIXME
image: #Image image: #Image
input: image // for compatibility with #Build
always: bool | *false always: bool | *false
@ -104,7 +104,10 @@ import (
message: string | *null message: string | *null
} }
output: { output?: {
// FIXME: hack for #Build compatibility
#Image
rootfs?: dagger.#FS & _exec.output rootfs?: dagger.#FS & _exec.output
files: [path=string]: { files: [path=string]: {
contents: string contents: string

363
europa-universe/examples/changelog.com/ci.cue
View File

@ -1,363 +0,0 @@
// STARTING POINT: https://docs.dagger.io/1012/ci
// + ../../../.circleci/config.yml
package ci
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
)
dagger.#Plan & {
// Receive things from client
input: {
directories: {
// App source code
app: {}
}
secrets: {
// Docker ID password
docker: {}
}
params: {
// Which Elixir base image to download
runtime_image: docker.#Ref | *"thechangelog/runtime:2021-05-29T10.17.12Z"
// Which test DB image to download
test_db_image: docker.#Ref | *"circleci/postgres:12.6"
}
}
// Send things to client
output: {
}
// Forward network services to and from the client
proxy: {
}
// Do things
actions: {
// Reuse in all mix commands
_appName: "changelog"
prod: {
assets: docker.#Build & {
steps: [
// 1. Start from dev assets :)
dev.assets,
// 2. Mix magical command
#mixRun & {
script: "mix phx.digest"
mix: {
env: "prod"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
// FIXME: remove copy-pasta
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: does this command need write access to node_modules cache?
concurrency: "readonly"
}
dest: "\(workdir)/node_modules"
}
}
]
}
}
dev: {
build: #mixBuild & {
"env": "dev"
app: "thechangelog"
base: input.params.runtime_image
source: input.directories.app.contents
}
assets: docker.#Build & {
steps: [
// 1. Start from dev runtime build
build,
// 2. Build web assets
#mixRun & {
mix: {
env: "dev"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
// FIXME: move this to a reusable def (yarn package? or private?)
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: will there be multiple writers?
concurrency: "locked"
}
dest: "\(workdir)/node_modules"
}
// FIXME: run 'yarn install' and 'yarn run compile' separately, with different caching?
// FIXME: can we reuse universe.dagger.io/yarn ???? 0:-)
script: "yarn install --frozen-lockfile && yarn run compile"
workdir: "/app/assets"
}
]
}
}
test: {
build: #mixBuild & {
"env": "test"
app: _appName
base: input.params.runtime_image
source: input.directories.app.contents
}
// Run tests
run: docker.#Run & {
image: build.output
script: "mix test"
// Don't cache running tests
// Just because we've tested a version before, doesn't mean we don't
// want to test it again.
always: true
}
db: {
// Pull test DB image
pull: docker.#Pull & {
source: input.params.test_db_image
}
// Run test DB
// FIXME: kill once no longer needed (when tests are done running)
run: docker.#Run & {
image: pull.output
}
}
}
}
}
// Helper to run mix correctly in a container
#mixRun: {
mix: {
app: string
env: string
depsCache?: "readonly" | "locked"
buildCache?: "readonly" | "locked"
}
"env": MIX_ENV: env
{
mix: depsCache: string
workdir: string
mounts: depsCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.depsCache
}
dest: "\(workdir)/deps"
}
} | {}
{
mix: buildCache: string
workdir: string
mounts: buildCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.buildCache
}
dest: "\(workdir)/deps"
}
} | {}
}
// FIXME: move into an elixir package
// Build an Elixir application with Mix
#mixBuild: {
// Ref to base image
// FIXME: spin out docker.#Build for max flexibility
// Perhaps implement as a custom docker.#Build step?
base: docker.#Ref
// App name (for cache scoping)
app: string
// Mix environment
env: string
// Application source code
source: dagger.#FS
docker.#Build & {
steps: [
// 1. Pull base image
docker.#Pull & {
source: base
},
// 2. Copy app source
docker.#Copy & {
contents: source
dest: "/app"
},
// 3. Download dependencies into deps cache
#mixRun & {
mix: {
"env": env
"app": app
depsCache: "locked"
}
workdir: "/app"
script: "mix deps.get"
},
// 4. Build!
#mixRun & {
mix: {
"env": env
"app": app
depsCache: "readonly"
buildCache: "locked"
}
workdir: "/app"
script: "mix do deps.compile, compile"
},
// 5. Set image config
// FIXME: how does this actually work? Does it mutate the field? Or is the field somehow
// prevented from being concrete? And if so, how?
docker.#Set & {
workdir: "/app"
"env": MIX_ENV: env
}
]
}
}
/////////////////
/////////////////
/////////////////
app: dagger.#Artifact
prod_dockerfile: string
docker_host: string
dockerhub_username: string
dockerhub_password: dagger.#Secret
// Keep this in sync with ../docker/Dockerfile.production
runtime_image_ref: dagger.#Input & {string | *"thechangelog/runtime:2021-05-29T10.17.12Z"}
prod_image_ref: dagger.#Input & {string | *"thechangelog/changelog.com:dagger"}
build_version: dagger.#Input & {string}
git_sha: dagger.#Input & {string}
git_author: dagger.#Input & {string}
app_version: dagger.#Input & {string}
build_url: dagger.#Input & {string}
// Keep this in sync with manifests/changelog/db.yml
test_db_image_ref: dagger.#Input & {string | *"circleci/postgres:12.6"}
test_db_container_name: "changelog_test_postgres"
run_test: dagger.#Input & {bool}
// STORY #######################################################################
//
// 1. Migrate from CircleCI to GitHub Actions
// - extract existing build pipeline into Dagger
// - run the pipeline locally
// - run the pipeline in GitHub Actions
//
// 2. Pipeline is up to 9x quicker (40s vs 370s)
// - optimistic branching, as pipelines were originally intended
// - use our own hardware - Linode g6-dedicated-8
// - predictable runs, no queuing
// - caching is buildkit layers
//
// 3. Open Telemetry integration out-of-the-box
// - visualise all steps in Jaeger UI
// CI PIPELINE OVERVIEW ########################################################
//
// deps_compile_test deps_compile_dev /--- deps_compile_prod
// | | | |
// v v | v
// test_cache assets_dev | image_prod_cache
// | | | |
// v v | |
// test_db_start -> test -> test_db_stop assets_prod <-----/ |
// | | |
// | v |
// | image_prod <-----------/
//
//....................................... TODO .................................
// | |
// | v
// \--------------------> image_prod_digest
//
// ========================== BEFORE | AFTER | CHANGE ===================================
// Test, build & push 370s | 40s | 9.25x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/520/workflows/fbb7c701-d25a-42c1-b42c-db514cd770b4
// + app compile 220s | 150s | 1.46x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/582/workflows/65500f3d-eccc-49da-9ab0-69846bc812a7
// + deps compile 480s | 190s | 2.52x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/532/workflows/94f5a339-52a1-45ba-b39b-1bbb69ed6488
//
// Uncached ???s | 465s | ?.??x |
//
// #############################################################################
deps_compile_prod: #deps_compile & {
args: {
MIX_ENV: "prod"
}
}
image_prod_cache: docker.#Build & {
source: deps_compile_prod
dockerfile: """
FROM \(runtime_image_ref)
COPY /app/ /app/
WORKDIR /app
RUN --mount=type=cache,id=deps,target=/mnt/app/deps,sharing=locked cp -Rp /mnt/app/deps/* /app/deps/
RUN --mount=type=cache,id=build_prod,target=/mnt/app/_build/prod,sharing=locked cp -Rp /mnt/app/_build/prod/* /app/_build/prod/
"""
}
image_prod: docker.#Command & {
host: docker_host
copy: {
"/tmp/app": from: os.#Dir & {
from: image_prod_cache
path: "/app"
}
"/tmp/app/priv/static": from: os.#Dir & {
from: assets_prod
path: "/app/priv/static"
}
}
env: {
GIT_AUTHOR: git_author
GIT_SHA: git_sha
APP_VERSION: app_version
BUILD_VERSION: build_version
BUILD_URL: build_url
DOCKERHUB_USERNAME: dockerhub_username
PROD_IMAGE_REF: prod_image_ref
}
files: "/tmp/Dockerfile": prod_dockerfile
secret: "/run/secrets/dockerhub_password": dockerhub_password
command: #"""
cd /tmp
docker build --build-arg GIT_AUTHOR="$GIT_AUTHOR" --build-arg GIT_SHA="$GIT_SHA" --build-arg APP_VERSION="$APP_VERSION" --build-arg BUILD_URL="$BUILD_URL" --tag "$PROD_IMAGE_REF" .
docker login --username "$DOCKERHUB_USERNAME" --password "$(cat /run/secrets/dockerhub_password)"
docker push "$PROD_IMAGE_REF" | tee docker.push.log
echo "$PROD_IMAGE_REF" > image.ref
awk '/digest/ { print $3 }' < docker.push.log > image.digest
"""#
}

134
europa-universe/examples/changelog.com/highlevel/ci.cue Normal file
View File

@ -0,0 +1,134 @@
package ci
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/docker"
"universe.dagger.io/examples/changelog.com/highlevel/elixir/mix"
)
dagger.#Plan & {
// Receive things from client
inputs: {
directories: {
// App source code
app: _
}
secrets: {
// Docker ID password
docker: _
}
params: {
// Which Elixir base image to download
runtime_image: docker.#Ref | *"thechangelog/runtime:2021-05-29T10.17.12Z"
// Which test DB image to download
test_db_image: docker.#Ref | *"circleci/postgres:12.6"
}
}
// Do things
actions: {
// Reuse in all mix commands
_appName: "changelog"
prod: assets: docker.#Build & {
steps: [
// 1. Start from dev assets :)
dev.assets,
// 2. Mix magical command
mix.#Run & {
script: "mix phx.digest"
mix: {
env: "prod"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
workdir: _
// FIXME: remove copy-pasta
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: does this command need write access to node_modules cache?
concurrency: "readonly"
}
dest: "\(workdir)/node_modules"
}
},
]
}
dev: {
build: mix.#Build & {
env: "dev"
app: "thechangelog"
base: inputs.params.runtime_image
source: inputs.directories.app.contents
}
assets: docker.#Build & {
steps: [
// 1. Start from dev runtime build
build,
// 2. Build web assets
mix.#Run & {
mix: {
env: "dev"
app: _appName
depsCache: "readonly"
buildCache: "readonly"
}
// FIXME: move this to a reusable def (yarn package? or private?)
mounts: nodeModules: {
contents: engine.#CacheDir & {
// FIXME: do we need an ID here?
id: "\(mix.app)_assets_node_modules"
// FIXME: will there be multiple writers?
concurrency: "locked"
}
dest: "\(workdir)/node_modules"
}
// FIXME: run 'yarn install' and 'yarn run compile' separately, with different caching?
// FIXME: can we reuse universe.dagger.io/yarn ???? 0:-)
script: "yarn install --frozen-lockfile && yarn run compile"
workdir: "/app/assets"
},
]
}
}
test: {
build: mix.#Build & {
env: "test"
app: _appName
base: inputs.params.runtime_image
source: inputs.directories.app.contents
}
// Run tests
run: docker.#Run & {
image: build.output
script: "mix test"
// Don't cache running tests
// Just because we've tested a version before, doesn't mean we don't
// want to test it again.
// FIXME: make this configurable
always: true
}
db: {
// Pull test DB image
pull: docker.#Pull & {
source: inputs.params.test_db_image
}
// Run test DB
// FIXME: kill once no longer needed (when tests are done running)
run: docker.#Run & {
image: pull.output
}
}
}
}
}

95
europa-universe/examples/changelog.com/highlevel/elixir/mix/mix.cue Normal file
View File

@ -0,0 +1,95 @@
package mix
import (
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/docker"
)
// Build an Elixir application with Mix
#Build: {
// Ref to base image
// FIXME: spin out docker.#Build for max flexibility
// Perhaps implement as a custom docker.#Build step?
base: docker.#Ref
// App name (for cache scoping)
app: string
// Mix environment
env: string
// Application source code
source: dagger.#FS
docker.#Build & {
steps: [
// 1. Pull base image
docker.#Pull & {
source: base
},
// 2. Copy app source
docker.#Copy & {
contents: source
dest: "/app"
},
// 3. Download dependencies into deps cache
#Run & {
mix: {
"env": env
"app": app
depsCache: "locked"
}
workdir: "/app"
script: "mix deps.get"
},
// 4. Build!
// FIXME: step 5 is to add image data, see issue 1339
#Run & {
mix: {
"env": env
"app": app
depsCache: "readonly"
buildCache: "locked"
}
workdir: "/app"
script: "mix do deps.compile, compile"
},
]
}
}
// Run mix correctly in a container
#Run: {
mix: {
app: string
env: string
depsCache?: "readonly" | "locked"
buildCache?: "readonly" | "locked"
}
docker.#Run
env: MIX_ENV: mix.env
{
mix: depsCache: string
workdir: string
mounts: depsCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.depsCache
}
dest: "\(workdir)/deps"
}
} | {}
{
mix: buildCache: string
workdir: string
mounts: buildCache: {
contents: engine.#CacheDir & {
id: "\(mix.app)_deps"
concurrency: mix.buildCache
}
dest: "\(workdir)/deps"
}
} | {}
}

116
europa-universe/examples/changelog.com/highlevel/old.cue Normal file
View File

@ -0,0 +1,116 @@
package ci
// app: dagger.#Artifact
// prod_dockerfile: string
// docker_host: string
// dockerhub_username: string
// dockerhub_password: dagger.#Secret
// // Keep this in sync with ../docker/Dockerfile.production
// runtime_image_ref: dagger.#Input & {string | *"thechangelog/runtime:2021-05-29T10.17.12Z"}
// prod_image_ref: dagger.#Input & {string | *"thechangelog/changelog.com:dagger"}
// build_version: dagger.#Input & {string}
// git_sha: dagger.#Input & {string}
// git_author: dagger.#Input & {string}
// app_version: dagger.#Input & {string}
// build_url: dagger.#Input & {string}
// // Keep this in sync with manifests/changelog/db.yml
// test_db_image_ref: dagger.#Input & {string | *"circleci/postgres:12.6"}
// test_db_container_name: "changelog_test_postgres"
// run_test: dagger.#Input & {bool}
//
// // STORY #######################################################################
// //
// // 1. Migrate from CircleCI to GitHub Actions
// // - extract existing build pipeline into Dagger
// // - run the pipeline locally
// // - run the pipeline in GitHub Actions
// //
// // 2. Pipeline is up to 9x quicker (40s vs 370s)
// // - optimistic branching, as pipelines were originally intended
// // - use our own hardware - Linode g6-dedicated-8
// // - predictable runs, no queuing
// // - caching is buildkit layers
// //
// // 3. Open Telemetry integration out-of-the-box
// // - visualise all steps in Jaeger UI
//
// // CI PIPELINE OVERVIEW ########################################################
// //
// // deps_compile_test deps_compile_dev /--- deps_compile_prod
// // | | | |
// // v v | v
// // test_cache assets_dev | image_prod_cache
// // | | | |
// // v v | |
// // test_db_start -> test -> test_db_stop assets_prod <-----/ |
// // | | |
// // | v |
// // | image_prod <-----------/
// //
// //....................................... TODO .................................
// // | |
// // | v
// // \--------------------> image_prod_digest
// //
// // ========================== BEFORE | AFTER | CHANGE ===================================
// // Test, build & push 370s | 40s | 9.25x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/520/workflows/fbb7c701-d25a-42c1-b42c-db514cd770b4
// // + app compile 220s | 150s | 1.46x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/582/workflows/65500f3d-eccc-49da-9ab0-69846bc812a7
// // + deps compile 480s | 190s | 2.52x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/532/workflows/94f5a339-52a1-45ba-b39b-1bbb69ed6488
// //
// // Uncached ???s | 465s | ?.??x |
// //
// // #############################################################################
//
//
//
//
// deps_compile_prod: #deps_compile & {
// args: {
// MIX_ENV: "prod"
// }
// }
//
// image_prod_cache: docker.#Build & {
// source: deps_compile_prod
// dockerfile: """
// FROM \(runtime_image_ref)
// COPY /app/ /app/
// WORKDIR /app
// RUN --mount=type=cache,id=deps,target=/mnt/app/deps,sharing=locked cp -Rp /mnt/app/deps/* /app/deps/
// RUN --mount=type=cache,id=build_prod,target=/mnt/app/_build/prod,sharing=locked cp -Rp /mnt/app/_build/prod/* /app/_build/prod/
// """
// }
//
// image_prod: docker.#Command & {
// host: docker_host
// copy: {
// "/tmp/app": from: os.#Dir & {
// from: image_prod_cache
// path: "/app"
// }
//
// "/tmp/app/priv/static": from: os.#Dir & {
// from: assets_prod
// path: "/app/priv/static"
// }
// }
// env: {
// GIT_AUTHOR: git_author
// GIT_SHA: git_sha
// APP_VERSION: app_version
// BUILD_VERSION: build_version
// BUILD_URL: build_url
// DOCKERHUB_USERNAME: dockerhub_username
// PROD_IMAGE_REF: prod_image_ref
// }
// files: "/tmp/Dockerfile": prod_dockerfile
// secret: "/run/secrets/dockerhub_password": dockerhub_password
// command: #"""
// cd /tmp
// docker build --build-arg GIT_AUTHOR="$GIT_AUTHOR" --build-arg GIT_SHA="$GIT_SHA" --build-arg APP_VERSION="$APP_VERSION" --build-arg BUILD_URL="$BUILD_URL" --tag "$PROD_IMAGE_REF" .
// docker login --username "$DOCKERHUB_USERNAME" --password "$(cat /run/secrets/dockerhub_password)"
// docker push "$PROD_IMAGE_REF" | tee docker.push.log
// echo "$PROD_IMAGE_REF" > image.ref
// awk '/digest/ { print $3 }' < docker.push.log > image.digest
// """#
// }

2
europa-universe/examples/changelog.com/lowlevel/README.md Normal file
View File

@ -0,0 +1,2 @@
Joel's low-level implementation goes here :)

2
europa-universe/examples/todoapp/base.cue
View File

@ -22,7 +22,7 @@ dagger.#DAG & {
remote: "https://github.com/mdn/todo-react" remote: "https://github.com/mdn/todo-react"
ref: "master" ref: "master"
} }
build: source: pull.checkout build: source: pull.output
} }
} }
} }

6
europa-universe/git/git.cue
View File

@ -1,8 +1,8 @@
package git package git
import ( import (
"dagger.io/dagger" "dagger.io/dagger/engine"
) )
#Pull: dagger.#GitPull #Pull: engine.#GitPull
#Push: dagger.#GitPush #Push: engine.#GitPush