From e8527ddcf5157a97cedbe40d157e17edbe84334c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 12 Feb 2021 22:37:41 +0000 Subject: [PATCH] No more runtime spec validation Signed-off-by: Solomon Hykes --- Makefile | 12 +- dagger/env.go | 65 +-------- dagger/gen.go | 110 --------------- dagger/gen.sh | 16 --- dagger/spec.go | 53 -------- dagger/spec_test.go | 59 -------- .../pkg/dagger.cloud/dagger/dagger.cue | 50 +------ examples/simple/simple.cue | 14 +- .../pkg/dagger.cloud/dagger/dagger.cue | 50 +------ .../pkg/dagger.cloud/netlify/netlify.cue | 126 ++++++++++++++++++ stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue | 64 +++++++++ 11 files changed, 209 insertions(+), 410 deletions(-) delete mode 100644 dagger/gen.go delete mode 100755 dagger/gen.sh delete mode 100644 dagger/spec.go delete mode 100644 dagger/spec_test.go rename dagger/spec.cue => stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue (51%) create mode 100644 stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue create mode 100644 stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue diff --git a/Makefile b/Makefile index 18e1fadc..fad4b94d 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,12 @@ .PHONY: all all: dagger -.PHONY: generate -generate: - @go generate ./dagger - .PHONY: dagger -dagger: generate +dagger: go build -o ./cmd/dagger/ ./cmd/dagger/ .PHONY: dagger -dagger-debug: generate +dagger-debug: go build -race -o ./cmd/dagger/dagger-debug ./cmd/dagger/ .PHONY: test @@ -22,7 +18,7 @@ cuefmt: @(cue fmt -s ./... && cue trim -s ./...) .PHONY: lint -lint: generate cuefmt +lint: cuefmt golangci-lint run @test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)" @test -z "$$(git status -s . | grep -e "^ M" | grep gen.go | cut -d ' ' -f3 | tee /dev/stderr)" @@ -35,4 +31,4 @@ integration: dagger-debug DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all update-examples: - cp ./dagger/spec.cue ./examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue + rsync -avH --delete ./stdlib/cue.mod/pkg/ ./examples/*/cue.mod/pkg/ diff --git a/dagger/env.go b/dagger/env.go index a6c96094..839f6812 100644 --- a/dagger/env.go +++ b/dagger/env.go @@ -2,7 +2,6 @@ package dagger import ( "context" - "os" "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" @@ -104,76 +103,14 @@ func (env *Env) Update(ctx context.Context, s Solver) error { if err != nil { return errors.Wrap(err, "base config") } - final, err := applySpec(base) - if err != nil { - return err - } // Commit return env.set( - final, + base, env.input, env.output, ) } -// Scan the env config for compute scripts, and merge the spec over them, -// for validation and default value expansion. -// This is done once when loading the env configuration, as opposed to dynamically -// during compute like in previous versions. Hopefully this will improve performance. -// -// Also note that performance was improved DRASTICALLY by splitting the #Component spec -// into individual #ComputableStruct, #ComputableString etc. It appears that it is massively -// faster to check for the type in Go, then apply the correct spec, than rely on a cue disjunction. -// -// FIXME: re-enable support for scalar types beyond string. -// -// FIXME: remove dependency on #Component def so it can be deprecated. -func applySpec(base *cc.Value) (*cc.Value, error) { - if os.Getenv("NO_APPLY_SPEC") != "" { - return base, nil - } - // Merge the spec to validate & expand buildkit scripts - computableStructs := []cue.Path{} - computableStrings := []cue.Path{} - base.Walk( - func(v *cc.Value) bool { - compute := v.Get("#dagger.compute") - if !compute.Exists() { - return true // keep scanning - } - if _, err := v.String(); err == nil { - // computable string - computableStrings = append(computableStrings, v.Path()) - return false - } - if _, err := v.Struct(); err == nil { - // computable struct - computableStructs = append(computableStructs, v.Path()) - return false - } - return false - }, - nil, - ) - structSpec := spec.Get("#ComputableStruct") - for _, target := range computableStructs { - newbase, err := base.MergePath(structSpec, target) - if err != nil { - return nil, err - } - base = newbase - } - stringSpec := spec.Get("#ComputableString") - for _, target := range computableStrings { - newbase, err := base.MergePath(stringSpec, target) - if err != nil { - return nil, err - } - base = newbase - } - return base, nil -} - func (env *Env) Base() *cc.Value { return env.base } diff --git a/dagger/gen.go b/dagger/gen.go deleted file mode 100644 index 1bcfb960..00000000 --- a/dagger/gen.go +++ /dev/null @@ -1,110 +0,0 @@ -package dagger - -// Generated by gen.sh. DO NOT EDIT. - -var DaggerSpec = ` -package dagger - -// A dagger component is a configuration value augmented -// by scripts defining how to compute it, present it to a user, -// encrypt it, etc. - -#ComputableStruct: { - #dagger: compute: [...#Op] - ... -} - -#ComputableString: { - string - #dagger: compute: [...#Op] -} - -#Component: { - // Match structs - #dagger: #ComponentConfig - ... -} | { - // Match embedded scalars - bool | int | float | string | bytes - #dagger: #ComponentConfig -} - -// The contents of a #dagger annotation -#ComponentConfig: { - // script to compute the value - compute?: #Script -} - -// Any component can be referenced as a directory, since -// every dagger script outputs a filesystem state (aka a directory) -#Dir: #Component - -#Script: [...#Op] - -// One operation in a script -#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir - -// Export a value from fs state to cue -#Export: { - do: "export" - // Source path in the container - source: string - format: "json" | "yaml" | *"string" -} - -#Local: { - do: "local" - dir: string - include: [...string] | *[] -} - -// FIXME: bring back load (more efficient than copy) - -#Load: { - do: "load" - from: #Component | #Script -} - -#Subdir: { - do: "subdir" - dir: string | *"/" -} - -#Exec: { - do: "exec" - args: [...string] - env?: [string]: string - always?: true | *false - dir: string | *"/" - mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript -} - -#MountTmp: "tmpfs" -#MountCache: "cache" -#MountComponent: { - from: #Component - path: string | *"/" -} -#MountScript: { - from: #Script - path: string | *"/" -} - -#FetchContainer: { - do: "fetch-container" - ref: string -} - -#FetchGit: { - do: "fetch-git" - remote: string - ref: string -} - -#Copy: { - do: "copy" - from: #Script | #Component - src: string | *"/" - dest: string | *"/" -} -` diff --git a/dagger/gen.sh b/dagger/gen.sh deleted file mode 100755 index 4f7e5a78..00000000 --- a/dagger/gen.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e -( -cat <<'EOF' -package dagger - -// Generated by gen.sh. DO NOT EDIT. - -var DaggerSpec = ` -EOF -cat spec.cue -cat <<'EOF' -` -EOF -) > gen.go diff --git a/dagger/spec.go b/dagger/spec.go deleted file mode 100644 index 3ea83c7b..00000000 --- a/dagger/spec.go +++ /dev/null @@ -1,53 +0,0 @@ -//go:generate sh gen.sh - -package dagger - -import ( - cueerrors "cuelang.org/go/cue/errors" - "github.com/pkg/errors" - - "dagger.cloud/go/dagger/cc" -) - -var ( - // Global shared dagger spec, generated from spec.cue - spec = NewSpec() -) - -// Cue spec validator -type Spec struct { - root *cc.Value -} - -func NewSpec() *Spec { - v, err := cc.Compile("spec.cue", DaggerSpec) - if err != nil { - panic(err) - } - if _, err := v.Struct(); err != nil { - panic(err) - } - return &Spec{ - root: v, - } -} - -// eg. Validate(op, "#Op") -func (s Spec) Validate(v *cc.Value, defpath string) error { - // Lookup def by name, eg. "#Script" or "#Copy" - // See dagger/spec.cue - def := s.root.Get(defpath) - if err := def.Fill(v); err != nil { - return errors.New(cueerrors.Details(err, nil)) - } - - return nil -} - -func (s Spec) Match(v *cc.Value, defpath string) bool { - return s.Validate(v, defpath) == nil -} - -func (s Spec) Get(target string) *cc.Value { - return s.root.Get(target) -} diff --git a/dagger/spec_test.go b/dagger/spec_test.go deleted file mode 100644 index d94e1d5e..00000000 --- a/dagger/spec_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package dagger - -import ( - "testing" - - "dagger.cloud/go/dagger/cc" -) - -func TestMatch(t *testing.T) { - var data = []struct { - Src string - Def string - }{ - { - Src: `do: "exec", args: ["echo", "hello"]`, - Def: "#Exec", - }, - { - Src: `do: "fetch-git", remote: "github.com/shykes/tests"`, - Def: "#FetchGit", - }, - } - for _, d := range data { - testMatch(t, d.Src, d.Def) - } -} - -// Test an example op for false positives and negatives -func testMatch(t *testing.T, src interface{}, def string) { - op := compile(t, src) - if def != "" { - if err := spec.Validate(op, def); err != nil { - t.Errorf("false negative: %s: %q: %s", def, src, err) - } - } - for _, cmpDef := range []string{ - "#Exec", - "#FetchGit", - "#FetchContainer", - "#Export", - "#Copy", - "#Local", - } { - if cmpDef == def { - continue - } - if err := spec.Validate(op, cmpDef); err == nil { - t.Errorf("false positive: %s: %q", cmpDef, src) - } - } -} - -func compile(t *testing.T, src interface{}) *cc.Value { - v, err := cc.Compile("", src) - if err != nil { - t.Fatal(err) - } - return v -} diff --git a/examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue b/examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue index a91955cf..8878bdbd 100644 --- a/examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue +++ b/examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue @@ -1,40 +1,9 @@ package dagger -// A dagger component is a configuration value augmented -// by scripts defining how to compute it, present it to a user, -// encrypt it, etc. - -#ComputableStruct: { - #dagger: compute: [...#Op] - ... -} - -#ComputableString: { - string - #dagger: compute: [...#Op] -} - -#Component: { - // Match structs - #dagger: #ComponentConfig - ... -} | { - // Match embedded scalars - bool | int | float | string | bytes - #dagger: #ComponentConfig -} - -// The contents of a #dagger annotation -#ComponentConfig: { - // script to compute the value - compute?: #Script -} // Any component can be referenced as a directory, since // every dagger script outputs a filesystem state (aka a directory) -#Dir: #Component - -#Script: [...#Op] +#Dir: #dagger: compute: [...#Op] // One operation in a script #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir @@ -57,7 +26,7 @@ package dagger #Load: { do: "load" - from: #Component | #Script + from: _ } #Subdir: { @@ -71,18 +40,7 @@ package dagger env?: [string]: string always?: true | *false dir: string | *"/" - mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript -} - -#MountTmp: "tmpfs" -#MountCache: "cache" -#MountComponent: { - from: #Component - path: string | *"/" -} -#MountScript: { - from: #Script - path: string | *"/" + mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" } } #FetchContainer: { @@ -98,7 +56,7 @@ package dagger #Copy: { do: "copy" - from: #Script | #Component + from: _ src: string | *"/" dest: string | *"/" } diff --git a/examples/simple/simple.cue b/examples/simple/simple.cue index e0a83809..37297f85 100644 --- a/examples/simple/simple.cue +++ b/examples/simple/simple.cue @@ -6,12 +6,12 @@ import ( "dagger.cloud/dagger" ) -let alpine={ +let alpine = { digest: "sha256:3c7497bf0c7af93428242d6176e8f7905f2201d8fc5861f45be7a346b5f23436" package: [string]: true | false | string #dagger: compute: [ { - do: "fetch-container" + do: "fetch-container" ref: "index.docker.io/alpine@\(digest)" }, for pkg, info in package { @@ -19,7 +19,7 @@ let alpine={ do: "exec" args: ["apk", "add", "-U", "--no-cache", pkg] } - if (info & string) != _|_ { + if (info & string) != _|_ { do: "exec" args: ["apk", "add", "-U", "--no-cache", "\(pkg)\(info)"] } @@ -41,13 +41,11 @@ www: { }, dagger.#Exec & { args: ["sh", "-c", "ls /src > /tmp/out"] - mount: "/src": { - from: source - } + mount: "/src": from: source }, dagger.#Export & { source: "/tmp/out" - } + }, ] } @@ -58,7 +56,7 @@ www: { #dagger: compute: [ { - do: "load" + do: "load" from: alpine }, dagger.#Exec & { diff --git a/dagger/spec.cue b/stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue similarity index 51% rename from dagger/spec.cue rename to stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue index a91955cf..8878bdbd 100644 --- a/dagger/spec.cue +++ b/stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue @@ -1,40 +1,9 @@ package dagger -// A dagger component is a configuration value augmented -// by scripts defining how to compute it, present it to a user, -// encrypt it, etc. - -#ComputableStruct: { - #dagger: compute: [...#Op] - ... -} - -#ComputableString: { - string - #dagger: compute: [...#Op] -} - -#Component: { - // Match structs - #dagger: #ComponentConfig - ... -} | { - // Match embedded scalars - bool | int | float | string | bytes - #dagger: #ComponentConfig -} - -// The contents of a #dagger annotation -#ComponentConfig: { - // script to compute the value - compute?: #Script -} // Any component can be referenced as a directory, since // every dagger script outputs a filesystem state (aka a directory) -#Dir: #Component - -#Script: [...#Op] +#Dir: #dagger: compute: [...#Op] // One operation in a script #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir @@ -57,7 +26,7 @@ package dagger #Load: { do: "load" - from: #Component | #Script + from: _ } #Subdir: { @@ -71,18 +40,7 @@ package dagger env?: [string]: string always?: true | *false dir: string | *"/" - mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript -} - -#MountTmp: "tmpfs" -#MountCache: "cache" -#MountComponent: { - from: #Component - path: string | *"/" -} -#MountScript: { - from: #Script - path: string | *"/" + mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" } } #FetchContainer: { @@ -98,7 +56,7 @@ package dagger #Copy: { do: "copy" - from: #Script | #Component + from: _ src: string | *"/" dest: string | *"/" } diff --git a/stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue b/stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue new file mode 100644 index 00000000..25a20093 --- /dev/null +++ b/stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue @@ -0,0 +1,126 @@ +package netlify + +import "dagger.cloud/dagger" + +// A Netlify account +#Account: { + // Use this Netlify account name + // (also referred to as "team" in the Netlify docs) + name: string | *"" + + // Netlify authentication token + token: string +} + +// A Netlify site +#Site: { + // Netlify account this site is attached to + account: #Account + + // Contents of the application to deploy + contents: dagger.#Dir + + // Deploy to this Netlify site + name: string + + // Host the site at this address + customDomain: string + + // Create the Netlify site if it doesn't exist? + create: bool | *true + + // Deployment url + url: { + string + + #dagger: compute: [ + dagger.#FetchContainer & { + ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930" + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"] + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "jq=1.6-r1"] + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "curl=7.74.0-r0"] + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"] + }, + dagger.#Exec & { + args: ["yarn", "global", "add", "netlify-cli@2.47.0"] + }, + dagger.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + code, + ] + env: { + NETLIFY_SITE_NAME: name + if (create) { + NETLIFY_SITE_CREATE: "1" + } + if customDomain != _|_ { + NETLIFY_DOMAIN: customDomain + } + NETLIFY_ACCOUNT: account.name + NETLIFY_AUTH_TOKEN: account.token + } + dir: "/src" + mount: "/src": from: contents + }, + dagger.#Export & { + source: "/url" + format: "string" + }, + ] + } +} + +// FIXME: this should be outside +let code = #""" + create_site() { + url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" + + response=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + $url \ + -d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" + ) + if [ $? -ne 0 ]; then + exit 1 + fi + + echo $response | jq -r '.site_id' + } + + site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + https://api.netlify.com/api/v1/sites\?filter\=all | \ + jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \ + ) + if [ -z "$site_id" ] ; then + if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then + echo "Site $NETLIFY_SITE_NAME does not exist" + exit 1 + fi + site_id=$(create_site) + if [ -z "$site_id" ]; then + echo "create site failed" + exit 1 + fi + fi + netlify deploy \ + --dir="$(pwd)" \ + --site="$site_id" \ + --prod \ + | tee /tmp/stdout + + /url + """# diff --git a/stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue b/stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue new file mode 100644 index 00000000..b73c4cc4 --- /dev/null +++ b/stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue @@ -0,0 +1,64 @@ +package yarn + +import ( + "dagger.cloud/dagger" +) + +// Yarn Script +#Script: { + // Source code of the javascript application + source: dagger.#Dir + + // Run this yarn script + run: string | *"build" + + // Read build output from this directory + // (path must be relative to working directory). + buildDirectory: string | *"build" + + // Set these environment variables during the build + env?: [string]: string + + #dagger: compute: [ + dagger.#FetchContainer & { + ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930" + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"] + }, + dagger.#Exec & { + args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"] + }, + dagger.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + """ + yarn install --production false + yarn run "$YARN_BUILD_SCRIPT" + mv "$YARN_BUILD_DIRECTORY" /build + """, + ] + if env != _|_ { + "env": env + } + "env": { + YARN_BUILD_SCRIPT: run + YARN_CACHE_FOLDER: "/cache/yarn" + YARN_BUILD_DIRECTORY: buildDirectory + } + dir: "/src" + mount: { + "/src": from: source + "/cache/yarn": dagger.#MountCache + } + }, + dagger.#Subdir & { + dir: "/build" + }, + ] +}