kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

No more runtime spec validation

Browse Source 
Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes 2021-02-12 22:37:41 +00:00
parent ec56160307
commit e8527ddcf5
11 changed files with 209 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@ -1,16 +1,12 @@
.PHONY: all .PHONY: all
all: dagger all: dagger
.PHONY: generate
generate:
@go generate ./dagger
.PHONY: dagger .PHONY: dagger
dagger: generate dagger:
go build -o ./cmd/dagger/ ./cmd/dagger/ go build -o ./cmd/dagger/ ./cmd/dagger/
.PHONY: dagger .PHONY: dagger
dagger-debug: generate dagger-debug:
go build -race -o ./cmd/dagger/dagger-debug ./cmd/dagger/ go build -race -o ./cmd/dagger/dagger-debug ./cmd/dagger/
.PHONY: test .PHONY: test
@ -22,7 +18,7 @@ cuefmt:
@(cue fmt -s ./... && cue trim -s ./...) @(cue fmt -s ./... && cue trim -s ./...)
.PHONY: lint .PHONY: lint
lint: generate cuefmt lint: cuefmt
golangci-lint run golangci-lint run
@test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)" @test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)"
@test -z "$$(git status -s . | grep -e "^ M" | grep gen.go | cut -d ' ' -f3 | tee /dev/stderr)" @test -z "$$(git status -s . | grep -e "^ M" | grep gen.go | cut -d ' ' -f3 | tee /dev/stderr)"
@ -35,4 +31,4 @@ integration: dagger-debug
DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all
update-examples: update-examples:
cp ./dagger/spec.cue ./examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue rsync -avH --delete ./stdlib/cue.mod/pkg/ ./examples/*/cue.mod/pkg/

65
dagger/env.go
View File

@ -2,7 +2,6 @@ package dagger
import ( import (
"context" "context"
"os"
"cuelang.org/go/cue" "cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow" cueflow "cuelang.org/go/tools/flow"
@ -104,76 +103,14 @@ func (env *Env) Update(ctx context.Context, s Solver) error {
if err != nil { if err != nil {
return errors.Wrap(err, "base config") return errors.Wrap(err, "base config")
} }
final, err := applySpec(base)
if err != nil {
return err
}
// Commit // Commit
return env.set( return env.set(
final, base,
env.input, env.input,
env.output, env.output,
) )
} }
// Scan the env config for compute scripts, and merge the spec over them,
// for validation and default value expansion.
// This is done once when loading the env configuration, as opposed to dynamically
// during compute like in previous versions. Hopefully this will improve performance.
//
// Also note that performance was improved DRASTICALLY by splitting the #Component spec
// into individual #ComputableStruct, #ComputableString etc. It appears that it is massively
// faster to check for the type in Go, then apply the correct spec, than rely on a cue disjunction.
//
// FIXME: re-enable support for scalar types beyond string.
//
// FIXME: remove dependency on #Component def so it can be deprecated.
func applySpec(base *cc.Value) (*cc.Value, error) {
if os.Getenv("NO_APPLY_SPEC") != "" {
return base, nil
}
// Merge the spec to validate & expand buildkit scripts
computableStructs := []cue.Path{}
computableStrings := []cue.Path{}
base.Walk(
func(v *cc.Value) bool {
compute := v.Get("#dagger.compute")
if !compute.Exists() {
return true // keep scanning
}
if _, err := v.String(); err == nil {
// computable string
computableStrings = append(computableStrings, v.Path())
return false
}
if _, err := v.Struct(); err == nil {
// computable struct
computableStructs = append(computableStructs, v.Path())
return false
}
return false
},
nil,
)
structSpec := spec.Get("#ComputableStruct")
for _, target := range computableStructs {
newbase, err := base.MergePath(structSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
stringSpec := spec.Get("#ComputableString")
for _, target := range computableStrings {
newbase, err := base.MergePath(stringSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
return base, nil
}
func (env *Env) Base() *cc.Value { func (env *Env) Base() *cc.Value {
return env.base return env.base
} }

110
dagger/gen.go
View File

@ -1,110 +0,0 @@
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json" | "yaml" | *"string"
}
#Local: {
do: "local"
dir: string
include: [...string] | *[]
}
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Subdir: {
do: "subdir"
dir: string | *"/"
}
#Exec: {
do: "exec"
args: [...string]
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
from: #Script | #Component
src: string | *"/"
dest: string | *"/"
}
`

16
dagger/gen.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
set -e
(
cat <<'EOF'
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
EOF
cat spec.cue
cat <<'EOF'
`
EOF
) > gen.go

53
dagger/spec.go
View File

@ -1,53 +0,0 @@
//go:generate sh gen.sh
package dagger
import (
cueerrors "cuelang.org/go/cue/errors"
"github.com/pkg/errors"
"dagger.cloud/go/dagger/cc"
)
var (
// Global shared dagger spec, generated from spec.cue
spec = NewSpec()
)
// Cue spec validator
type Spec struct {
root *cc.Value
}
func NewSpec() *Spec {
v, err := cc.Compile("spec.cue", DaggerSpec)
if err != nil {
panic(err)
}
if _, err := v.Struct(); err != nil {
panic(err)
}
return &Spec{
root: v,
}
}
// eg. Validate(op, "#Op")
func (s Spec) Validate(v *cc.Value, defpath string) error {
// Lookup def by name, eg. "#Script" or "#Copy"
// See dagger/spec.cue
def := s.root.Get(defpath)
if err := def.Fill(v); err != nil {
return errors.New(cueerrors.Details(err, nil))
}
return nil
}
func (s Spec) Match(v *cc.Value, defpath string) bool {
return s.Validate(v, defpath) == nil
}
func (s Spec) Get(target string) *cc.Value {
return s.root.Get(target)
}

59
dagger/spec_test.go
View File

@ -1,59 +0,0 @@
package dagger
import (
"testing"
"dagger.cloud/go/dagger/cc"
)
func TestMatch(t *testing.T) {
var data = []struct {
Src string
Def string
}{
{
Src: `do: "exec", args: ["echo", "hello"]`,
Def: "#Exec",
},
{
Src: `do: "fetch-git", remote: "github.com/shykes/tests"`,
Def: "#FetchGit",
},
}
for _, d := range data {
testMatch(t, d.Src, d.Def)
}
}
// Test an example op for false positives and negatives
func testMatch(t *testing.T, src interface{}, def string) {
op := compile(t, src)
if def != "" {
if err := spec.Validate(op, def); err != nil {
t.Errorf("false negative: %s: %q: %s", def, src, err)
}
}
for _, cmpDef := range []string{
"#Exec",
"#FetchGit",
"#FetchContainer",
"#Export",
"#Copy",
"#Local",
} {
if cmpDef == def {
continue
}
if err := spec.Validate(op, cmpDef); err == nil {
t.Errorf("false positive: %s: %q", cmpDef, src)
}
}
}
func compile(t *testing.T, src interface{}) *cc.Value {
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
return v
}

50
examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue
View File

@ -1,40 +1,9 @@
package dagger package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since // Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory) // every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component #Dir: #dagger: compute: [...#Op]
#Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
@ -57,7 +26,7 @@ package dagger
#Load: { #Load: {
do: "load" do: "load"
from: #Component | #Script from: _
} }
#Subdir: { #Subdir: {
@ -71,18 +40,7 @@ package dagger
env?: [string]: string env?: [string]: string
always?: true | *false always?: true | *false
dir: string | *"/" dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" }
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
} }
#FetchContainer: { #FetchContainer: {
@ -98,7 +56,7 @@ package dagger
#Copy: { #Copy: {
do: "copy" do: "copy"
from: #Script | #Component from: _
src: string | *"/" src: string | *"/"
dest: string | *"/" dest: string | *"/"
} }

14
examples/simple/simple.cue
View File

@ -6,12 +6,12 @@ import (
"dagger.cloud/dagger" "dagger.cloud/dagger"
) )
let alpine={ let alpine = {
digest: "sha256:3c7497bf0c7af93428242d6176e8f7905f2201d8fc5861f45be7a346b5f23436" digest: "sha256:3c7497bf0c7af93428242d6176e8f7905f2201d8fc5861f45be7a346b5f23436"
package: [string]: true | false | string package: [string]: true | false | string
#dagger: compute: [ #dagger: compute: [
{ {
do: "fetch-container" do: "fetch-container"
ref: "index.docker.io/alpine@\(digest)" ref: "index.docker.io/alpine@\(digest)"
}, },
for pkg, info in package { for pkg, info in package {
@ -19,7 +19,7 @@ let alpine={
do: "exec" do: "exec"
args: ["apk", "add", "-U", "--no-cache", pkg] args: ["apk", "add", "-U", "--no-cache", pkg]
} }
if (info & string) != _|_ { if (info & string) != _|_ {
do: "exec" do: "exec"
args: ["apk", "add", "-U", "--no-cache", "\(pkg)\(info)"] args: ["apk", "add", "-U", "--no-cache", "\(pkg)\(info)"]
} }
@ -41,13 +41,11 @@ www: {
}, },
dagger.#Exec & { dagger.#Exec & {
args: ["sh", "-c", "ls /src > /tmp/out"] args: ["sh", "-c", "ls /src > /tmp/out"]
mount: "/src": { mount: "/src": from: source
from: source
}
}, },
dagger.#Export & { dagger.#Export & {
source: "/tmp/out" source: "/tmp/out"
} },
] ]
} }
@ -58,7 +56,7 @@ www: {
#dagger: compute: [ #dagger: compute: [
{ {
do: "load" do: "load"
from: alpine from: alpine
}, },
dagger.#Exec & { dagger.#Exec & {

50
dagger/spec.cue → stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue
View File

@ -1,40 +1,9 @@
package dagger package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since // Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory) // every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component #Dir: #dagger: compute: [...#Op]
#Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
@ -57,7 +26,7 @@ package dagger
#Load: { #Load: {
do: "load" do: "load"
from: #Component | #Script from: _
} }
#Subdir: { #Subdir: {
@ -71,18 +40,7 @@ package dagger
env?: [string]: string env?: [string]: string
always?: true | *false always?: true | *false
dir: string | *"/" dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" }
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
} }
#FetchContainer: { #FetchContainer: {
@ -98,7 +56,7 @@ package dagger
#Copy: { #Copy: {
do: "copy" do: "copy"
from: #Script | #Component from: _
src: string | *"/" src: string | *"/"
dest: string | *"/" dest: string | *"/"
} }

126
stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue Normal file
View File

@ -0,0 +1,126 @@
package netlify
import "dagger.cloud/dagger"
// A Netlify account
#Account: {
// Use this Netlify account name
// (also referred to as "team" in the Netlify docs)
name: string | *""
// Netlify authentication token
token: string
}
// A Netlify site
#Site: {
// Netlify account this site is attached to
account: #Account
// Contents of the application to deploy
contents: dagger.#Dir
// Deploy to this Netlify site
name: string
// Host the site at this address
customDomain: string
// Create the Netlify site if it doesn't exist?
create: bool | *true
// Deployment url
url: {
string
#dagger: compute: [
dagger.#FetchContainer & {
ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930"
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "jq=1.6-r1"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "curl=7.74.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"]
},
dagger.#Exec & {
args: ["yarn", "global", "add", "netlify-cli@2.47.0"]
},
dagger.#Exec & {
args: [
"/bin/bash",
"--noprofile",
"--norc",
"-eo",
"pipefail",
"-c",
code,
]
env: {
NETLIFY_SITE_NAME: name
if (create) {
NETLIFY_SITE_CREATE: "1"
}
if customDomain != _|_ {
NETLIFY_DOMAIN: customDomain
}
NETLIFY_ACCOUNT: account.name
NETLIFY_AUTH_TOKEN: account.token
}
dir: "/src"
mount: "/src": from: contents
},
dagger.#Export & {
source: "/url"
format: "string"
},
]
}
}
// FIXME: this should be outside
let code = #"""
create_site() {
url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
response=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-X POST -H "Content-Type: application/json" \
$url \
-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}"
)
if [ $? -ne 0 ]; then
exit 1
fi
echo $response | jq -r '.site_id'
}
site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
https://api.netlify.com/api/v1/sites\?filter\=all | \
jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
)
if [ -z "$site_id" ] ; then
if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
echo "Site $NETLIFY_SITE_NAME does not exist"
exit 1
fi
site_id=$(create_site)
if [ -z "$site_id" ]; then
echo "create site failed"
exit 1
fi
fi
netlify deploy \
--dir="$(pwd)" \
--site="$site_id" \
--prod \
| tee /tmp/stdout
</tmp/stdout sed -n -e 's/^Website URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n' > /url
"""#

64
stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue Normal file
View File

@ -0,0 +1,64 @@
package yarn
import (
"dagger.cloud/dagger"
)
// Yarn Script
#Script: {
// Source code of the javascript application
source: dagger.#Dir
// Run this yarn script
run: string | *"build"
// Read build output from this directory
// (path must be relative to working directory).
buildDirectory: string | *"build"
// Set these environment variables during the build
env?: [string]: string
#dagger: compute: [
dagger.#FetchContainer & {
ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930"
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"]
},
dagger.#Exec & {
args: [
"/bin/bash",
"--noprofile",
"--norc",
"-eo",
"pipefail",
"-c",
"""
yarn install --production false
yarn run "$YARN_BUILD_SCRIPT"
mv "$YARN_BUILD_DIRECTORY" /build
""",
]
if env != _|_ {
"env": env
}
"env": {
YARN_BUILD_SCRIPT: run
YARN_CACHE_FOLDER: "/cache/yarn"
YARN_BUILD_DIRECTORY: buildDirectory
}
dir: "/src"
mount: {
"/src": from: source
"/cache/yarn": dagger.#MountCache
}
},
dagger.#Subdir & {
dir: "/build"
},
]
}