changelog.com europa experiment
Signed-off-by: Solomon Hykes <solomon@dagger.io>
This commit is contained in:
Solomon Hykes
Solomon Hykes
parent
9a4c8fb845
commit
6d4d10a002
363
europa-universe/examples/changelog.com/ci.cue
Normal file
363
europa-universe/examples/changelog.com/ci.cue
Normal file
@ -0,0 +1,363 @@
|
||||
// STARTING POINT: https://docs.dagger.io/1012/ci
|
||||
// + ../../../.circleci/config.yml
|
||||
package ci
|
||||
|
||||
import (
|
||||
"dagger.io/dagger"
|
||||
"universe.dagger.io/docker"
|
||||
)
|
||||
|
||||
dagger.#Plan & {
|
||||
// Receive things from client
|
||||
input: {
|
||||
directories: {
|
||||
// App source code
|
||||
app: {}
|
||||
}
|
||||
secrets: {
|
||||
// Docker ID password
|
||||
docker: {}
|
||||
}
|
||||
params: {
|
||||
// Which Elixir base image to download
|
||||
runtime_image: docker.#Ref | *"thechangelog/runtime:2021-05-29T10.17.12Z"
|
||||
// Which test DB image to download
|
||||
test_db_image: docker.#Ref | *"circleci/postgres:12.6"
|
||||
}
|
||||
}
|
||||
|
||||
// Send things to client
|
||||
output: {
|
||||
}
|
||||
|
||||
// Forward network services to and from the client
|
||||
proxy: {
|
||||
}
|
||||
|
||||
// Do things
|
||||
actions: {
|
||||
|
||||
// Reuse in all mix commands
|
||||
_appName: "changelog"
|
||||
|
||||
prod: {
|
||||
|
||||
assets: docker.#Build & {
|
||||
steps: [
|
||||
// 1. Start from dev assets :)
|
||||
dev.assets,
|
||||
// 2. Mix magical command
|
||||
#mixRun & {
|
||||
script: "mix phx.digest"
|
||||
mix: {
|
||||
env: "prod"
|
||||
app: _appName
|
||||
depsCache: "readonly"
|
||||
buildCache: "readonly"
|
||||
}
|
||||
// FIXME: remove copy-pasta
|
||||
mounts: nodeModules: {
|
||||
contents: engine.#CacheDir & {
|
||||
// FIXME: do we need an ID here?
|
||||
id: "\(mix.app)_assets_node_modules"
|
||||
// FIXME: does this command need write access to node_modules cache?
|
||||
concurrency: "readonly"
|
||||
}
|
||||
dest: "\(workdir)/node_modules"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
dev: {
|
||||
build: #mixBuild & {
|
||||
"env": "dev"
|
||||
app: "thechangelog"
|
||||
base: input.params.runtime_image
|
||||
source: input.directories.app.contents
|
||||
}
|
||||
|
||||
assets: docker.#Build & {
|
||||
steps: [
|
||||
// 1. Start from dev runtime build
|
||||
build,
|
||||
// 2. Build web assets
|
||||
#mixRun & {
|
||||
mix: {
|
||||
env: "dev"
|
||||
app: _appName
|
||||
depsCache: "readonly"
|
||||
buildCache: "readonly"
|
||||
}
|
||||
// FIXME: move this to a reusable def (yarn package? or private?)
|
||||
mounts: nodeModules: {
|
||||
contents: engine.#CacheDir & {
|
||||
// FIXME: do we need an ID here?
|
||||
id: "\(mix.app)_assets_node_modules"
|
||||
// FIXME: will there be multiple writers?
|
||||
concurrency: "locked"
|
||||
}
|
||||
dest: "\(workdir)/node_modules"
|
||||
}
|
||||
// FIXME: run 'yarn install' and 'yarn run compile' separately, with different caching?
|
||||
// FIXME: can we reuse universe.dagger.io/yarn ???? 0:-)
|
||||
script: "yarn install --frozen-lockfile && yarn run compile"
|
||||
workdir: "/app/assets"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
test: {
|
||||
|
||||
build: #mixBuild & {
|
||||
"env": "test"
|
||||
app: _appName
|
||||
base: input.params.runtime_image
|
||||
source: input.directories.app.contents
|
||||
}
|
||||
|
||||
// Run tests
|
||||
run: docker.#Run & {
|
||||
image: build.output
|
||||
script: "mix test"
|
||||
// Don't cache running tests
|
||||
// Just because we've tested a version before, doesn't mean we don't
|
||||
// want to test it again.
|
||||
always: true
|
||||
}
|
||||
|
||||
db: {
|
||||
// Pull test DB image
|
||||
pull: docker.#Pull & {
|
||||
source: input.params.test_db_image
|
||||
}
|
||||
|
||||
// Run test DB
|
||||
// FIXME: kill once no longer needed (when tests are done running)
|
||||
run: docker.#Run & {
|
||||
image: pull.output
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// Helper to run mix correctly in a container
|
||||
#mixRun: {
|
||||
mix: {
|
||||
app: string
|
||||
env: string
|
||||
depsCache?: "readonly" | "locked"
|
||||
buildCache?: "readonly" | "locked"
|
||||
}
|
||||
"env": MIX_ENV: env
|
||||
{
|
||||
mix: depsCache: string
|
||||
workdir: string
|
||||
mounts: depsCache: {
|
||||
contents: engine.#CacheDir & {
|
||||
id: "\(mix.app)_deps"
|
||||
concurrency: mix.depsCache
|
||||
}
|
||||
dest: "\(workdir)/deps"
|
||||
}
|
||||
} | {}
|
||||
{
|
||||
mix: buildCache: string
|
||||
workdir: string
|
||||
mounts: buildCache: {
|
||||
contents: engine.#CacheDir & {
|
||||
id: "\(mix.app)_deps"
|
||||
concurrency: mix.buildCache
|
||||
}
|
||||
dest: "\(workdir)/deps"
|
||||
}
|
||||
} | {}
|
||||
}
|
||||
|
||||
// FIXME: move into an elixir package
|
||||
// Build an Elixir application with Mix
|
||||
#mixBuild: {
|
||||
// Ref to base image
|
||||
// FIXME: spin out docker.#Build for max flexibility
|
||||
// Perhaps implement as a custom docker.#Build step?
|
||||
base: docker.#Ref
|
||||
|
||||
// App name (for cache scoping)
|
||||
app: string
|
||||
|
||||
// Mix environment
|
||||
env: string
|
||||
|
||||
// Application source code
|
||||
source: dagger.#FS
|
||||
|
||||
docker.#Build & {
|
||||
steps: [
|
||||
// 1. Pull base image
|
||||
docker.#Pull & {
|
||||
source: base
|
||||
},
|
||||
// 2. Copy app source
|
||||
docker.#Copy & {
|
||||
contents: source
|
||||
dest: "/app"
|
||||
},
|
||||
// 3. Download dependencies into deps cache
|
||||
#mixRun & {
|
||||
mix: {
|
||||
"env": env
|
||||
"app": app
|
||||
depsCache: "locked"
|
||||
}
|
||||
workdir: "/app"
|
||||
script: "mix deps.get"
|
||||
},
|
||||
// 4. Build!
|
||||
#mixRun & {
|
||||
mix: {
|
||||
"env": env
|
||||
"app": app
|
||||
depsCache: "readonly"
|
||||
buildCache: "locked"
|
||||
}
|
||||
workdir: "/app"
|
||||
script: "mix do deps.compile, compile"
|
||||
},
|
||||
// 5. Set image config
|
||||
// FIXME: how does this actually work? Does it mutate the field? Or is the field somehow
|
||||
// prevented from being concrete? And if so, how?
|
||||
docker.#Set & {
|
||||
workdir: "/app"
|
||||
"env": MIX_ENV: env
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/////////////////
|
||||
/////////////////
|
||||
/////////////////
|
||||
|
||||
app: dagger.#Artifact
|
||||
prod_dockerfile: string
|
||||
docker_host: string
|
||||
dockerhub_username: string
|
||||
dockerhub_password: dagger.#Secret
|
||||
// Keep this in sync with ../docker/Dockerfile.production
|
||||
runtime_image_ref: dagger.#Input & {string | *"thechangelog/runtime:2021-05-29T10.17.12Z"}
|
||||
prod_image_ref: dagger.#Input & {string | *"thechangelog/changelog.com:dagger"}
|
||||
build_version: dagger.#Input & {string}
|
||||
git_sha: dagger.#Input & {string}
|
||||
git_author: dagger.#Input & {string}
|
||||
app_version: dagger.#Input & {string}
|
||||
build_url: dagger.#Input & {string}
|
||||
// Keep this in sync with manifests/changelog/db.yml
|
||||
test_db_image_ref: dagger.#Input & {string | *"circleci/postgres:12.6"}
|
||||
test_db_container_name: "changelog_test_postgres"
|
||||
run_test: dagger.#Input & {bool}
|
||||
|
||||
// STORY #######################################################################
|
||||
//
|
||||
// 1. Migrate from CircleCI to GitHub Actions
|
||||
// - extract existing build pipeline into Dagger
|
||||
// - run the pipeline locally
|
||||
// - run the pipeline in GitHub Actions
|
||||
//
|
||||
// 2. Pipeline is up to 9x quicker (40s vs 370s)
|
||||
// - optimistic branching, as pipelines were originally intended
|
||||
// - use our own hardware - Linode g6-dedicated-8
|
||||
// - predictable runs, no queuing
|
||||
// - caching is buildkit layers
|
||||
//
|
||||
// 3. Open Telemetry integration out-of-the-box
|
||||
// - visualise all steps in Jaeger UI
|
||||
|
||||
// CI PIPELINE OVERVIEW ########################################################
|
||||
//
|
||||
// deps_compile_test deps_compile_dev /--- deps_compile_prod
|
||||
// | | | |
|
||||
// v v | v
|
||||
// test_cache assets_dev | image_prod_cache
|
||||
// | | | |
|
||||
// v v | |
|
||||
// test_db_start -> test -> test_db_stop assets_prod <-----/ |
|
||||
// | | |
|
||||
// | v |
|
||||
// | image_prod <-----------/
|
||||
//
|
||||
//....................................... TODO .................................
|
||||
// | |
|
||||
// | v
|
||||
// \--------------------> image_prod_digest
|
||||
//
|
||||
// ========================== BEFORE | AFTER | CHANGE ===================================
|
||||
// Test, build & push 370s | 40s | 9.25x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/520/workflows/fbb7c701-d25a-42c1-b42c-db514cd770b4
|
||||
// + app compile 220s | 150s | 1.46x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/582/workflows/65500f3d-eccc-49da-9ab0-69846bc812a7
|
||||
// + deps compile 480s | 190s | 2.52x | https://app.circleci.com/pipelines/github/thechangelog/changelog.com/532/workflows/94f5a339-52a1-45ba-b39b-1bbb69ed6488
|
||||
//
|
||||
// Uncached ???s | 465s | ?.??x |
|
||||
//
|
||||
// #############################################################################
|
||||
|
||||
|
||||
|
||||
|
||||
deps_compile_prod: #deps_compile & {
|
||||
args: {
|
||||
MIX_ENV: "prod"
|
||||
}
|
||||
}
|
||||
|
||||
image_prod_cache: docker.#Build & {
|
||||
source: deps_compile_prod
|
||||
dockerfile: """
|
||||
FROM \(runtime_image_ref)
|
||||
COPY /app/ /app/
|
||||
WORKDIR /app
|
||||
RUN --mount=type=cache,id=deps,target=/mnt/app/deps,sharing=locked cp -Rp /mnt/app/deps/* /app/deps/
|
||||
RUN --mount=type=cache,id=build_prod,target=/mnt/app/_build/prod,sharing=locked cp -Rp /mnt/app/_build/prod/* /app/_build/prod/
|
||||
"""
|
||||
}
|
||||
|
||||
image_prod: docker.#Command & {
|
||||
host: docker_host
|
||||
copy: {
|
||||
"/tmp/app": from: os.#Dir & {
|
||||
from: image_prod_cache
|
||||
path: "/app"
|
||||
}
|
||||
|
||||
"/tmp/app/priv/static": from: os.#Dir & {
|
||||
from: assets_prod
|
||||
path: "/app/priv/static"
|
||||
}
|
||||
}
|
||||
env: {
|
||||
GIT_AUTHOR: git_author
|
||||
GIT_SHA: git_sha
|
||||
APP_VERSION: app_version
|
||||
BUILD_VERSION: build_version
|
||||
BUILD_URL: build_url
|
||||
DOCKERHUB_USERNAME: dockerhub_username
|
||||
PROD_IMAGE_REF: prod_image_ref
|
||||
}
|
||||
files: "/tmp/Dockerfile": prod_dockerfile
|
||||
secret: "/run/secrets/dockerhub_password": dockerhub_password
|
||||
command: #"""
|
||||
cd /tmp
|
||||
docker build --build-arg GIT_AUTHOR="$GIT_AUTHOR" --build-arg GIT_SHA="$GIT_SHA" --build-arg APP_VERSION="$APP_VERSION" --build-arg BUILD_URL="$BUILD_URL" --tag "$PROD_IMAGE_REF" .
|
||||
docker login --username "$DOCKERHUB_USERNAME" --password "$(cat /run/secrets/dockerhub_password)"
|
||||
docker push "$PROD_IMAGE_REF" | tee docker.push.log
|
||||
echo "$PROD_IMAGE_REF" > image.ref
|
||||
awk '/digest/ { print $3 }' < docker.push.log > image.digest
|
||||
"""#
|
||||
}
|
||||
Reference in New Issue
Block a user