kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #116 from blocklayerhq/slim2

Browse Source 
Remove runtime spec validation
This commit is contained in:
Andrea Luzzardi 2021-02-12 16:12:59 -08:00 committed by GitHub
commit df35cdfc4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 209 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@ -1,16 +1,12 @@
.PHONY: all
all: dagger
.PHONY: generate
generate:
@go generate ./dagger
.PHONY: dagger
dagger: generate
dagger:
go build -o ./cmd/dagger/ ./cmd/dagger/
.PHONY: dagger
dagger-debug: generate
dagger-debug:
go build -race -o ./cmd/dagger/dagger-debug ./cmd/dagger/
.PHONY: test
@ -22,7 +18,7 @@ cuefmt:
@(cue fmt -s ./... && cue trim -s ./...)
.PHONY: lint
lint: generate cuefmt
lint: cuefmt
golangci-lint run
@test -z "$$(git status -s . | grep -e "^ M" | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)"
@test -z "$$(git status -s . | grep -e "^ M" | grep gen.go | cut -d ' ' -f3 | tee /dev/stderr)"
@ -35,4 +31,4 @@ integration: dagger-debug
DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all
update-examples:
cp ./dagger/spec.cue ./examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue
rsync -avH --delete ./stdlib/cue.mod/pkg/ ./examples/*/cue.mod/pkg/

65
dagger/env.go
View File

@ -2,7 +2,6 @@ package dagger
import (
"context"
"os"
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
@ -104,76 +103,14 @@ func (env *Env) Update(ctx context.Context, s Solver) error {
if err != nil {
return errors.Wrap(err, "base config")
}
final, err := applySpec(base)
if err != nil {
return err
}
// Commit
return env.set(
final,
base,
env.input,
env.output,
)
}
// Scan the env config for compute scripts, and merge the spec over them,
// for validation and default value expansion.
// This is done once when loading the env configuration, as opposed to dynamically
// during compute like in previous versions. Hopefully this will improve performance.
//
// Also note that performance was improved DRASTICALLY by splitting the #Component spec
// into individual #ComputableStruct, #ComputableString etc. It appears that it is massively
// faster to check for the type in Go, then apply the correct spec, than rely on a cue disjunction.
//
// FIXME: re-enable support for scalar types beyond string.
//
// FIXME: remove dependency on #Component def so it can be deprecated.
func applySpec(base *cc.Value) (*cc.Value, error) {
if os.Getenv("NO_APPLY_SPEC") != "" {
return base, nil
}
// Merge the spec to validate & expand buildkit scripts
computableStructs := []cue.Path{}
computableStrings := []cue.Path{}
base.Walk(
func(v *cc.Value) bool {
compute := v.Get("#dagger.compute")
if !compute.Exists() {
return true // keep scanning
}
if _, err := v.String(); err == nil {
// computable string
computableStrings = append(computableStrings, v.Path())
return false
}
if _, err := v.Struct(); err == nil {
// computable struct
computableStructs = append(computableStructs, v.Path())
return false
}
return false
},
nil,
)
structSpec := spec.Get("#ComputableStruct")
for _, target := range computableStructs {
newbase, err := base.MergePath(structSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
stringSpec := spec.Get("#ComputableString")
for _, target := range computableStrings {
newbase, err := base.MergePath(stringSpec, target)
if err != nil {
return nil, err
}
base = newbase
}
return base, nil
}
func (env *Env) Base() *cc.Value {
return env.base
}

110
dagger/gen.go
View File

@ -1,110 +0,0 @@
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
// Export a value from fs state to cue
#Export: {
do: "export"
// Source path in the container
source: string
format: "json" | "yaml" | *"string"
}
#Local: {
do: "local"
dir: string
include: [...string] | *[]
}
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Subdir: {
do: "subdir"
dir: string | *"/"
}
#Exec: {
do: "exec"
args: [...string]
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
}
#FetchContainer: {
do: "fetch-container"
ref: string
}
#FetchGit: {
do: "fetch-git"
remote: string
ref: string
}
#Copy: {
do: "copy"
from: #Script | #Component
src: string | *"/"
dest: string | *"/"
}
`

16
dagger/gen.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
set -e
(
cat <<'EOF'
package dagger
// Generated by gen.sh. DO NOT EDIT.
var DaggerSpec = `
EOF
cat spec.cue
cat <<'EOF'
`
EOF
) > gen.go

53
dagger/spec.go
View File

@ -1,53 +0,0 @@
//go:generate sh gen.sh
package dagger
import (
cueerrors "cuelang.org/go/cue/errors"
"github.com/pkg/errors"
"dagger.cloud/go/dagger/cc"
)
var (
// Global shared dagger spec, generated from spec.cue
spec = NewSpec()
)
// Cue spec validator
type Spec struct {
root *cc.Value
}
func NewSpec() *Spec {
v, err := cc.Compile("spec.cue", DaggerSpec)
if err != nil {
panic(err)
}
if _, err := v.Struct(); err != nil {
panic(err)
}
return &Spec{
root: v,
}
}
// eg. Validate(op, "#Op")
func (s Spec) Validate(v *cc.Value, defpath string) error {
// Lookup def by name, eg. "#Script" or "#Copy"
// See dagger/spec.cue
def := s.root.Get(defpath)
if err := def.Fill(v); err != nil {
return errors.New(cueerrors.Details(err, nil))
}
return nil
}
func (s Spec) Match(v *cc.Value, defpath string) bool {
return s.Validate(v, defpath) == nil
}
func (s Spec) Get(target string) *cc.Value {
return s.root.Get(target)
}

59
dagger/spec_test.go
View File

@ -1,59 +0,0 @@
package dagger
import (
"testing"
"dagger.cloud/go/dagger/cc"
)
func TestMatch(t *testing.T) {
var data = []struct {
Src string
Def string
}{
{
Src: `do: "exec", args: ["echo", "hello"]`,
Def: "#Exec",
},
{
Src: `do: "fetch-git", remote: "github.com/shykes/tests"`,
Def: "#FetchGit",
},
}
for _, d := range data {
testMatch(t, d.Src, d.Def)
}
}
// Test an example op for false positives and negatives
func testMatch(t *testing.T, src interface{}, def string) {
op := compile(t, src)
if def != "" {
if err := spec.Validate(op, def); err != nil {
t.Errorf("false negative: %s: %q: %s", def, src, err)
}
}
for _, cmpDef := range []string{
"#Exec",
"#FetchGit",
"#FetchContainer",
"#Export",
"#Copy",
"#Local",
} {
if cmpDef == def {
continue
}
if err := spec.Validate(op, cmpDef); err == nil {
t.Errorf("false positive: %s: %q", cmpDef, src)
}
}
}
func compile(t *testing.T, src interface{}) *cc.Value {
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
return v
}

50
examples/simple/cue.mod/pkg/dagger.cloud/dagger/dagger.cue
View File

@ -1,40 +1,9 @@
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
#Dir: #dagger: compute: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
@ -57,7 +26,7 @@ package dagger
#Load: {
do: "load"
from: #Component | #Script
from: _
}
#Subdir: {
@ -71,18 +40,7 @@ package dagger
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" }
}
#FetchContainer: {
@ -98,7 +56,7 @@ package dagger
#Copy: {
do: "copy"
from: #Script | #Component
from: _
src: string | *"/"
dest: string | *"/"
}

6
examples/simple/simple.cue
View File

@ -41,13 +41,11 @@ www: {
},
dagger.#Exec & {
args: ["sh", "-c", "ls /src > /tmp/out"]
mount: "/src": {
from: source
}
mount: "/src": from: source
},
dagger.#Export & {
source: "/tmp/out"
}
},
]
}

50
dagger/spec.cue → stdlib/cue.mod/pkg/dagger.cloud/dagger/dagger.cue
View File

@ -1,40 +1,9 @@
package dagger
// A dagger component is a configuration value augmented
// by scripts defining how to compute it, present it to a user,
// encrypt it, etc.
#ComputableStruct: {
#dagger: compute: [...#Op]
...
}
#ComputableString: {
string
#dagger: compute: [...#Op]
}
#Component: {
// Match structs
#dagger: #ComponentConfig
...
} | {
// Match embedded scalars
bool | int | float | string | bytes
#dagger: #ComponentConfig
}
// The contents of a #dagger annotation
#ComponentConfig: {
// script to compute the value
compute?: #Script
}
// Any component can be referenced as a directory, since
// every dagger script outputs a filesystem state (aka a directory)
#Dir: #Component
#Script: [...#Op]
#Dir: #dagger: compute: [...#Op]
// One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load | #Subdir
@ -57,7 +26,7 @@ package dagger
#Load: {
do: "load"
from: #Component | #Script
from: _
}
#Subdir: {
@ -71,18 +40,7 @@ package dagger
env?: [string]: string
always?: true | *false
dir: string | *"/"
mount: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
}
#MountTmp: "tmpfs"
#MountCache: "cache"
#MountComponent: {
from: #Component
path: string | *"/"
}
#MountScript: {
from: #Script
path: string | *"/"
mount: [string]: "tmp" | "cache" | { from: _, path: string | *"/" }
}
#FetchContainer: {
@ -98,7 +56,7 @@ package dagger
#Copy: {
do: "copy"
from: #Script | #Component
from: _
src: string | *"/"
dest: string | *"/"
}

126
stdlib/cue.mod/pkg/dagger.cloud/netlify/netlify.cue Normal file
View File

@ -0,0 +1,126 @@
package netlify
import "dagger.cloud/dagger"
// A Netlify account
#Account: {
// Use this Netlify account name
// (also referred to as "team" in the Netlify docs)
name: string | *""
// Netlify authentication token
token: string
}
// A Netlify site
#Site: {
// Netlify account this site is attached to
account: #Account
// Contents of the application to deploy
contents: dagger.#Dir
// Deploy to this Netlify site
name: string
// Host the site at this address
customDomain: string
// Create the Netlify site if it doesn't exist?
create: bool | *true
// Deployment url
url: {
string
#dagger: compute: [
dagger.#FetchContainer & {
ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930"
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "jq=1.6-r1"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "curl=7.74.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"]
},
dagger.#Exec & {
args: ["yarn", "global", "add", "netlify-cli@2.47.0"]
},
dagger.#Exec & {
args: [
"/bin/bash",
"--noprofile",
"--norc",
"-eo",
"pipefail",
"-c",
code,
]
env: {
NETLIFY_SITE_NAME: name
if (create) {
NETLIFY_SITE_CREATE: "1"
}
if customDomain != _|_ {
NETLIFY_DOMAIN: customDomain
}
NETLIFY_ACCOUNT: account.name
NETLIFY_AUTH_TOKEN: account.token
}
dir: "/src"
mount: "/src": from: contents
},
dagger.#Export & {
source: "/url"
format: "string"
},
]
}
}
// FIXME: this should be outside
let code = #"""
create_site() {
url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
response=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-X POST -H "Content-Type: application/json" \
$url \
-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}"
)
if [ $? -ne 0 ]; then
exit 1
fi
echo $response | jq -r '.site_id'
}
site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
https://api.netlify.com/api/v1/sites\?filter\=all | \
jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
)
if [ -z "$site_id" ] ; then
if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
echo "Site $NETLIFY_SITE_NAME does not exist"
exit 1
fi
site_id=$(create_site)
if [ -z "$site_id" ]; then
echo "create site failed"
exit 1
fi
fi
netlify deploy \
--dir="$(pwd)" \
--site="$site_id" \
--prod \
| tee /tmp/stdout
</tmp/stdout sed -n -e 's/^Website URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n' > /url
"""#

64
stdlib/cue.mod/pkg/dagger.cloud/yarn/yarn.cue Normal file
View File

@ -0,0 +1,64 @@
package yarn
import (
"dagger.cloud/dagger"
)
// Yarn Script
#Script: {
// Source code of the javascript application
source: dagger.#Dir
// Run this yarn script
run: string | *"build"
// Read build output from this directory
// (path must be relative to working directory).
buildDirectory: string | *"build"
// Set these environment variables during the build
env?: [string]: string
#dagger: compute: [
dagger.#FetchContainer & {
ref: "alpine@sha256:08d6ca16c60fe7490c03d10dc339d9fd8ea67c6466dea8d558526b1330a85930"
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "bash=5.1.0-r0"]
},
dagger.#Exec & {
args: ["apk", "add", "-U", "--no-cache", "yarn=1.22.10-r0"]
},
dagger.#Exec & {
args: [
"/bin/bash",
"--noprofile",
"--norc",
"-eo",
"pipefail",
"-c",
"""
yarn install --production false
yarn run "$YARN_BUILD_SCRIPT"
mv "$YARN_BUILD_DIRECTORY" /build
""",
]
if env != _|_ {
"env": env
}
"env": {
YARN_BUILD_SCRIPT: run
YARN_CACHE_FOLDER: "/cache/yarn"
YARN_BUILD_DIRECTORY: buildDirectory
}
dir: "/src"
mount: {
"/src": from: source
"/cache/yarn": dagger.#MountCache
}
},
dagger.#Subdir & {
dir: "/build"
},
]
}