Move prototype 69-dagger-archon to top-level

Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes
2020-12-29 18:45:16 -08:00
commit 30f75da114
42 changed files with 3955 additions and 0 deletions

View File

@@ -0,0 +1,23 @@
# ACME platform
Welcome to the acme-platform repository. It contains everything you need to start developing and shipping improvements
to the ACME Clothing Store.
For information or support, contact the ACME Platform team: platform@acme.infralabs.io
# Things you can do with ACME platform
## Pre-merge or post-merge QA
## Create a personal dev environment
## Cross-team integration testing
## Sales demos
## End-to-end product reviews
## Testing infrastructure changes
## Deploying to production (REQUIRES SPECIAL PRIVILEGES, talk to your SRE)

View File

@@ -0,0 +1,38 @@
// ACME platform: everything you need to develop and ship improvements to
// the ACME clothing store.
package acme
import (
"dagger.cloud/dagger"
"dagger.cloud/netlify"
"dagger.cloud/aws/ecs"
"dagger.cloud/microstaging"
)
// Website on netlify
www: netlify & {
domain: string | *defaultDomain
// By default, use a generated microstaging.io domain
// for easy environments on demand.
let defaultDomain=microstaging.#Domain & {
token: _
prefix: "www.acme"
}
}
// API deployed on ECS
api: ecs & {
domain: _ | *defaultDomain
let defaultDomain = microstaging.#Domain & {
token: _
prefix: "api.acme"
}
}
// Database on RDS
db: rds & {
engine: "postgresql"
}

View File

@@ -0,0 +1,24 @@
package alpine
#Image: {
version: string | *"latest"
packages: [...string]
#dag: {
do: [
{
//
//
// fetch alpine
},
{
for _, pkg in packages {
}
}
]
}
}

View File

@@ -0,0 +1 @@
module: "acme.infralabs.io/acme"

View File

@@ -0,0 +1,104 @@
package dagger
// A DAG is the basic unit of programming in dagger.
// It is a special kind of program which runs as a pipeline of computing nodes running in parallel,
// instead of a sequence of operations to be run by a single node.
//
// It is a powerful way to automate various parts of an application delivery workflow:
// build, test, deploy, generate configuration, enforce policies, publish artifacts, etc.
//
// The DAG architecture has many benefits:
// - Because DAGs are made of nodes executing in parallel, they are easy to scale.
// - Because all inputs and outputs are snapshotted and content-addressed, DAGs
// can easily be made repeatable, can be cached aggressively, and can be replayed
// at will.
// - Because nodes are executed by the same container engine as docker-build, DAGs
// can be developed using any language or technology capable of running in a docker.
// Dockerfiles and docker images are natively supported for maximum compatibility.
//
// - Because DAGs are programmed declaratively with a powerful configuration language,
// they are much easier to test, debug and refactor than traditional programming languages.
//
// To execute a DAG, the dagger runtime JIT-compiles it to a low-level format called
// llb, and executes it with buildkit.
// Think of buildkit as a specialized VM for running compute graphs; and dagger as
// a complete programming environment for that VM.
//
// The tradeoff for all those wonderful features is that a DAG architecture cannot be used
// for all software: only software than can be run as a pipeline.
//
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#Component: #dagger: {
// script to compute the value
compute?: #Script
terminal?: {
// Display a message when opening a terminal session
greeting?: string
command: [string]: #Script
}
// Configure how the component is incorporated to user settings.
// Configure how the end-user can configure this component
settings?: {
// If not specified, scrape from comments
title?: string
description?: string
// Disable user input, even if incomplete?
hidden: true | *false
ui: _ // insert here something which can be compiled to react-jsonschema-form
// Show the cue default value to the user, as a default input value?
showDefault: true | *false
// Insert information needed by:
// 1) clients to encrypt
// ie. web wizard, cli
// 2) middleware to implement deicphering in the cuellb pipeline
// eg. integration with clcoud KMS, Vault...
//
// 3) connectors to make sure secrets are preserved
encrypt?: {
pubkey: string
cipher: string
}
}
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
#Op: #Fetch | #Export | #Run | #Local
// Export a value from fs state to cue
#Export: ["export", string, "json"|"yaml"|"string"|"number"|"boolean"]
#Run: #runNoOpts | #runWithOpts
#runNoOpts: ["run", ...string]
#runWithOpts: ["run", #RunOpts, ...string]
#RunOpts: {
mount?: string: "tmpfs" | "cache" | { from: #Component|#Script }
env: [string]: string
dir: string | *"/"
always: true | *false
}
#Local: ["local", string]
#Fetch: #FetchGit | #FetchContainer
#FetchContainer: ["fetch", "container", string]
#FetchGit: ["fetch", "git", string, string]
#TestScript: #Script & [
["fetch", "container", "alpine:latest"],
["run", "echo", "hello", "world"]
]

View File

@@ -0,0 +1,63 @@
package netlify
import (
".../alpine"
)
auth: {
#dag: {
encrypted: true
do: [
{
action: "fetch"
type: "docker"
source: "alpine"
},
{
action: "push"
}
]
}
{
username: string
password: string
} | {
// FIXME: enrypted!
token: string
}
}
name: string
domain?: string
// FIXME: directory!
source: bl.#Dir
let base = alpine.#Image & {
version: "foo"
packages: ["rsync", "npm", "openssh"]
}
// Netlify site ID
id: {
info1: string
info2: string
#dag: {
// run code to fetch id from netlify API
from: base
do: [
{
action: "run"
command: ["netlify-get-id", name, "-o", "/netlify-id.json"]
}
]
export: json: "/netlify-id.json"
}
}
url: string

View File

@@ -0,0 +1,216 @@
// Custom netlify package
// ACME platform team <platform@acme.infralabs.io>
//
// TODO: upstream to dagger standard library.
package netlify
import (
"dagger.cloud/dagger"
)
// Netlify API token
token: {
#dag: {
encrypt: cipher: "..."
}
string
}
// Netlify site name
name?: string
// Source directory to deploy
source: dagger.#Dir
let apply={
#dag: {
from: alpine.#Base
do: [
["run", "npm", "install", "netlify-cli", "-g"],
[
"copy",
[
"fetch", "git", "https://github.com/shykes/tests", "netlify-scripts",
], "/", "/src",
]
// 2. fetch custom netlify scripts & iunstall
// 3. get ID from name; create if doesn't exist
// 4. deploy (via builder)
]
command: {
debug: {
from: base
do: ["run", "sh", "-c", """
env && find /netlify
"""]
}
}
}
}
apply
deployedDir: {
#dag: {
from: apply
export: dir: "/netlify/content"
}
}
// Netlify site ID
ID: {
string
#dag: {
from: apply
export: string: "/netlify/site-id"
}
}
url: {
string
#dag: {
from: apply
export: string: "/netlify/url"
}
}
// Example of short-form cuellb pipeline
// 1. single-op pipeline can omit the array
// 2. action encoded in first key, instead of `action: ` field
// 3. op may implement short-form,
// in this case: `run: [...string]` instead of `run: { command: [...string] }`
do: run: ["ntlfy-get-site-id", name, "-o", "/netlify/site-id"]
// Declarative export from container, instead of awkward `readFile` pseudo-op
export: string: "/netlify/site-id"
}
}
// Configuration presets
preset: {
*"html" | "react" | "custom"
#dag: {
settings: {
markup: select: {
"Static HTML site (no build)": "html"
"ReactJS app built with npm": "react"
"Custom builder": "custom"
}
}
}
}
// Custom builder
// Default: no build, deploy as-is.
builder: {
in: dagger.#Dir & source
out: dagger.#Dir
if preset == "html" {
// Pass-through builder that does nothing
out: in
}
if preset == "react" {
let app = reactjs.#App & {
source: in
}
out: app.build
}
...
}
scripts: {
dagger.#Directory | *latestScripts
let latestScripts = {
#dag: {
do: {
action: "fetch"
type: "git"
source: "https://github.com/shykes/tests"
ref: "netlify-scripts"
}
}
export: dir: "/"
}
}
// This is configurable for dev mode, but hide it from end users.
#dag: settings: hidden: true
}
// Version of the netlify CLI to use
cliVersion: string | *latestCLIVersion
let latestCLIVersion = {
string
#dag: {
from: base
do: run: ["sh", "-c", "npm show netlify-cli dist-tags.latest > /latest-cli-version"]
export: string: "/latest-cli-version"
}
}
// Custom container to run netlify commands + wrappers
let base=alpine.#Base & {
package: {
npm: true
curl: true
}
}
let runner = {
#dag: {
from: base
do: [
{
run: "npm", "install
action: "run"
command: ["npm", "install", "-g", "netlify-cli@" + cliVersion]
},
{
// YOU ARE HERE
// incorporate "netify scripts from personal github" pattern from other POC
}
}
}
url: {
string
#dag: {
from: runner
do: run: {
command: #"""
netlify deploy
--dir="$(pwd)" \
--auth="$(cat /netlify/token)" \
--site="${NETLIFY_SITE_ID}" \
--message="Blocklayer 'netlify deploy'" \
--prod \
| tee /tmp/stdout
curl \
-i -X POST \
-H "Authorization: Bearer $(cat /netlify/token)" \
"https://api.netlify.com/api/v1/sites/${NETLIFY_SITE_ID}/ssl"
"""#
mount: {
"/netlify/token": token
"/netlify/source": builder.out
}
dir: "/netlify/source"
env: {
NETLIFY_SITE_ID: ID
}
}
}
}

View File

@@ -0,0 +1,89 @@
package netlify
#dag: {
do: [
{
action: "fetch"
type: "container"
repository: "alpine"
tag: "latest"
},
{
action: "run"
command: "apk add ..."
},
{
action: "copy"
from: [
{
action: "fetch"
type: "git"
repo: "https://github.com/shykes/stuff"
}
]
source: "/"
dest: "/src"
},
]
}
// Name of the netlify site
name: {
string
#dag: {
}
}
// ID of the netlify site
// FIXME: compute
id: {
string
#dag: {
from: ...
do: [
action: "run"
command: ["netlify-get-id", name, "-o", "/netlify-id.txt"]
]
export: string: "/netlify-id.txt"
}
}
// API token
// FIXME: encrypt secret!
token: {
#encrypt: {
pubkey: _
cipher: _
}
string
}
// FIXME: how to receive a directory?
source: bl.#Dir
// Domain of the Netlify site
domain?: string
// FIXME: compute
url: {
#dag: {
do: [
// ...
{
action: "run"
command: "netlify deploy"
dir: "/src"
mount: "/src": source
}
]
}
string
}

View File

@@ -0,0 +1,3 @@

View File

@@ -0,0 +1,46 @@
package alpine
// Default version pinned to digest. Manually updated.
let defaultDigest="sha256:3c7497bf0c7af93428242d6176e8f7905f2201d8fc5861f45be7a346b5f23436"
ref: string
// Match a combination of inputs 'version' and 'digest':
*{
// no version, no digest:
ref: "index.docker.io/alpine@\(defaultDigest)"
} | {
// version, no digest
version: string
ref: "alpine:\(version)"
} | {
// digest, no version
digest: string
ref: "alpine@\(digest)"
} | {
// version and digest
version: string
digest: string
ref: "alpine:\(version)@\(digest)"
}
// Packages to install
package: [string]: true | false | string
#dagger: compute: [
{
do: "fetch-container"
"ref": ref
},
for pkg, info in package {
if (info & true) != _|_ {
do: "exec"
args: ["apk", "add", "-U", "--no-cache", pkg]
}
if (info & string) != _|_ {
do: "exec"
args: ["apk", "add", "-U", "--no-cache", "\(pkg)\(info)"]
}
},
]

View File

@@ -0,0 +1,145 @@
package dagger
// A DAG is the basic unit of programming in dagger.
// It is a special kind of program which runs as a pipeline of computing nodes running in parallel,
// instead of a sequence of operations to be run by a single node.
//
// It is a powerful way to automate various parts of an application delivery workflow:
// build, test, deploy, generate configuration, enforce policies, publish artifacts, etc.
//
// The DAG architecture has many benefits:
// - Because DAGs are made of nodes executing in parallel, they are easy to scale.
// - Because all inputs and outputs are snapshotted and content-addressed, DAGs
// can easily be made repeatable, can be cached aggressively, and can be replayed
// at will.
// - Because nodes are executed by the same container engine as docker-build, DAGs
// can be developed using any language or technology capable of running in a docker.
// Dockerfiles and docker images are natively supported for maximum compatibility.
//
// - Because DAGs are programmed declaratively with a powerful configuration language,
// they are much easier to test, debug and refactor than traditional programming languages.
//
// To execute a DAG, the dagger runtime JIT-compiles it to a low-level format called
// llb, and executes it with buildkit.
// Think of buildkit as a specialized VM for running compute graphs; and dagger as
// a complete programming environment for that VM.
//
// The tradeoff for all those wonderful features is that a DAG architecture cannot be used
// for all software: only software than can be run as a pipeline.
//
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#Component: {
#dagger: {
// script to compute the value
compute?: #Script
terminal?: {
// Display a message when opening a terminal session
greeting?: string
command: [string]: #Script
}
// Configure how the component is incorporated to user settings.
// Configure how the end-user can configure this component
settings?: {
// If not specified, scrape from comments
title?: string
description?: string
// Disable user input, even if incomplete?
hidden: true | *false
ui: _ // insert here something which can be compiled to react-jsonschema-form
// Show the cue default value to the user, as a default input value?
showDefault: true | *false
// Insert information needed by:
// 1) clients to encrypt
// ie. web wizard, cli
// 2) middleware to implement deicphering in the cuellb pipeline
// eg. integration with clcoud KMS, Vault...
//
// 3) connectors to make sure secrets are preserved
encrypt?: {
pubkey: string
cipher: string
}
}
}
...
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
// #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Copy | #Load
#Op: #FetchContainer | #Export | #Exec
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json"|"yaml"|*"string"|"number"|"boolean"
}
#Load: #LoadComponent| #LoadScript
#LoadComponent: {
do: "load"
from: #Component
}
#LoadScript: {
do: "load"
from: #Script
}
#Exec: {
do: "exec"
args: [...string]
mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
env: [string]: string
dir: string | *"/"
always: true | *false
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
input: #Component
path: string | *"/"
}
#MountScript: {
input: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
input: #Script | #Component
src: string | *"/"
dest: string | *"/"
}
#TestScript: #Script & [
{ do: "fetch-container", ref: "alpine:latest" },
{ do: "exec", args: ["echo", "hello", "world" ], env: DEBUG: "1" }
]

View File

@@ -0,0 +1,63 @@
package example
import (
"dagger.cloud/alpine"
)
test: {
string
#dagger: compute: [
{ do: "load", from: alpine },
{
do: "copy"
from: [
{ do: "fetch-container", ref: alpine.ref },
]
dest: "/src"
},
{
do: "exec"
dir: "/src"
args: ["sh", "-c", """
ls -l > /tmp/out
"""
]
},
{
do: "export"
source: "/tmp/out"
format: "string"
}
]
}
www: {
// Domain where the site will be deployed (user input)
domain: string
// URL after deployment (computed)
url: {
string & =~ "https://.*"
#dagger: {
compute: [
{ do: "load", from: alpine },
{
do: "exec"
args: ["sh", "-c",
"""
echo 'deploying to netlify (not really)'
echo 'https://\(domain)/foo' > /tmp/out
"""
]
},
{
do: "export"
source: "/tmp/out"
format: "string"
}
]
}
}
}

View File

@@ -0,0 +1,3 @@
package example
www: domain: "www.foobar.com"

View File

@@ -0,0 +1 @@
module: "acme.infralabs.io/acme"

View File

@@ -0,0 +1,46 @@
package alpine
// Default version pinned to digest. Manually updated.
let defaultDigest="sha256:3c7497bf0c7af93428242d6176e8f7905f2201d8fc5861f45be7a346b5f23436"
ref: string
// Match a combination of inputs 'version' and 'digest':
*{
// no version, no digest:
ref: "index.docker.io/alpine@\(defaultDigest)"
} | {
// version, no digest
version: string
ref: "alpine:\(version)"
} | {
// digest, no version
digest: string
ref: "alpine@\(digest)"
} | {
// version and digest
version: string
digest: string
ref: "alpine:\(version)@\(digest)"
}
// Packages to install
package: [string]: true | false | string
#dagger: compute: [
{
do: "fetch-container"
"ref": ref
},
for pkg, info in package {
if (info & true) != _|_ {
do: "exec"
args: ["apk", "add", "-U", "--no-cache", pkg]
}
if (info & string) != _|_ {
do: "exec"
args: ["apk", "add", "-U", "--no-cache", "\(pkg)\(info)"]
}
},
]

View File

@@ -0,0 +1,145 @@
package dagger
// A DAG is the basic unit of programming in dagger.
// It is a special kind of program which runs as a pipeline of computing nodes running in parallel,
// instead of a sequence of operations to be run by a single node.
//
// It is a powerful way to automate various parts of an application delivery workflow:
// build, test, deploy, generate configuration, enforce policies, publish artifacts, etc.
//
// The DAG architecture has many benefits:
// - Because DAGs are made of nodes executing in parallel, they are easy to scale.
// - Because all inputs and outputs are snapshotted and content-addressed, DAGs
// can easily be made repeatable, can be cached aggressively, and can be replayed
// at will.
// - Because nodes are executed by the same container engine as docker-build, DAGs
// can be developed using any language or technology capable of running in a docker.
// Dockerfiles and docker images are natively supported for maximum compatibility.
//
// - Because DAGs are programmed declaratively with a powerful configuration language,
// they are much easier to test, debug and refactor than traditional programming languages.
//
// To execute a DAG, the dagger runtime JIT-compiles it to a low-level format called
// llb, and executes it with buildkit.
// Think of buildkit as a specialized VM for running compute graphs; and dagger as
// a complete programming environment for that VM.
//
// The tradeoff for all those wonderful features is that a DAG architecture cannot be used
// for all software: only software than can be run as a pipeline.
//
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#Component: {
#dagger: {
// script to compute the value
compute?: #Script
terminal?: {
// Display a message when opening a terminal session
greeting?: string
command: [string]: #Script
}
// Configure how the component is incorporated to user settings.
// Configure how the end-user can configure this component
settings?: {
// If not specified, scrape from comments
title?: string
description?: string
// Disable user input, even if incomplete?
hidden: true | *false
ui: _ // insert here something which can be compiled to react-jsonschema-form
// Show the cue default value to the user, as a default input value?
showDefault: true | *false
// Insert information needed by:
// 1) clients to encrypt
// ie. web wizard, cli
// 2) middleware to implement deicphering in the cuellb pipeline
// eg. integration with clcoud KMS, Vault...
//
// 3) connectors to make sure secrets are preserved
encrypt?: {
pubkey: string
cipher: string
}
}
}
...
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
// #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Copy | #Load
#Op: #FetchContainer | #Export | #Exec
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json"|"yaml"|*"string"|"number"|"boolean"
}
#Load: #LoadComponent| #LoadScript
#LoadComponent: {
do: "load"
from: #Component
}
#LoadScript: {
do: "load"
from: #Script
}
#Exec: {
do: "exec"
args: [...string]
mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
env: [string]: string
dir: string | *"/"
always: true | *false
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
input: #Component
path: string | *"/"
}
#MountScript: {
input: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
input: #Script | #Component
src: string | *"/"
dest: string | *"/"
}
#TestScript: #Script & [
{ do: "fetch-container", ref: "alpine:latest" },
{ do: "exec", args: ["echo", "hello", "world" ], env: DEBUG: "1" }
]

View File

@@ -0,0 +1,33 @@
// ACME platform: everything you need to develop and ship improvements to
// the ACME clothing store.
package acme
import (
"dagger.cloud/alpine"
)
let base=alpine & {
package: {
bash: ">3.0"
rsync: true
}
}
www: {
source: {
#dagger: input: true
}
host: string
url: {
string
#dagger: compute: [
{ do: "load", from: base },
{ do: "exec", args: ["sh", "-c", "echo -n 'https://\(host)/foo' > /tmp/out"] },
{ do: "export", format: "string", source: "/tmp/out" },
]
}
}

View File

@@ -0,0 +1,3 @@
package acme
www: host: "acme.infralabs.io"