kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Refactor

Browse Source 
Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes 2021-01-05 00:37:29 -08:00
parent 22500ad9db
commit 3162ca0991
25 changed files with 1664 additions and 1296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/dagger/cmd/compute.go
View File

@ -3,7 +3,6 @@ package cmd
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"dagger.cloud/go/dagger" "dagger.cloud/go/dagger"
"dagger.cloud/go/dagger/ui" "dagger.cloud/go/dagger/ui"
@ -16,32 +15,20 @@ var computeCmd = &cobra.Command{
Args: cobra.ExactArgs(1), Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
ctx := context.TODO() ctx := context.TODO()
c, err := dagger.NewClient(ctx, "") // FIXME: boot and bootdir should be config fields, not args
c, err := dagger.NewClient(ctx, "", "", args[0])
if err != nil { if err != nil {
ui.Fatal(err) ui.Fatal(err)
} }
target := args[0] // FIXME: configure which config to compute (duh)
if target == "-" { // FIXME: configure inputs
ui.Info("Assembling config from stdin\n")
// FIXME: include cue.mod/pkg from *somewhere* so stdin config can import
if err := c.SetConfig(os.Stdin); err != nil {
ui.Fatal(err)
}
} else {
ui.Info("Assembling config from %q\n", target)
if err := c.SetConfig(target); err != nil {
ui.Fatal(err)
}
}
ui.Info("Running") ui.Info("Running")
output, err := c.Run(ctx, "compute") output, err := c.Compute(ctx)
if err != nil { if err != nil {
ui.Fatal(err) ui.Fatal(err)
} }
ui.Info("Processing output") ui.Info("Processing output")
// output.Print(os.Stdout)
fmt.Println(output.JSON()) fmt.Println(output.JSON())
// FIXME: write computed values back to env dir
}, },
} }

265
dagger/client.go
View File

@ -5,7 +5,6 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"os" "os"
"strings" "strings"
"time" "time"
@ -15,14 +14,10 @@ import (
// Cue // Cue
"cuelang.org/go/cue" "cuelang.org/go/cue"
cueerrors "cuelang.org/go/cue/errors"
cueformat "cuelang.org/go/cue/format"
// buildkit // buildkit
bk "github.com/moby/buildkit/client" bk "github.com/moby/buildkit/client"
_ "github.com/moby/buildkit/client/connhelper/dockercontainer" _ "github.com/moby/buildkit/client/connhelper/dockercontainer"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
// docker output // docker output
"github.com/containerd/console" "github.com/containerd/console"
@ -32,21 +27,34 @@ import (
const ( const (
defaultBuildkitHost = "docker-container://buildkitd" defaultBuildkitHost = "docker-container://buildkitd"
bkConfigKey = "context" bkBootKey = "boot"
bkInputKey = ":dagger:input:" bkInputKey = "input"
bkActionKey = ":dagger:action:"
defaultBootDir = "."
// FIXME: rename to defaultConfig ?
defaultBootScript = `
bootdir: string | *"."
bootscript: [
{
do: "local"
dir: bootdir
include: ["*.cue", "cue.mod"]
},
]
`
) )
type Client struct { type Client struct {
c *bk.Client c *bk.Client
inputs map[string]llb.State
localdirs map[string]string localdirs map[string]string
boot string
BKFrontend bkgw.BuildFunc bootdir string
input string
} }
func NewClient(ctx context.Context, host string) (*Client, error) { func NewClient(ctx context.Context, host, boot, bootdir string) (*Client, error) {
// buildkit client // buildkit client
if host == "" { if host == "" {
host = os.Getenv("BUILDKIT_HOST") host = os.Getenv("BUILDKIT_HOST")
@ -54,107 +62,59 @@ func NewClient(ctx context.Context, host string) (*Client, error) {
if host == "" { if host == "" {
host = defaultBuildkitHost host = defaultBuildkitHost
} }
if boot == "" {
boot = defaultBootScript
}
if bootdir == "" {
bootdir = defaultBootDir
}
c, err := bk.New(ctx, host) c, err := bk.New(ctx, host)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "buildkit client") return nil, errors.Wrap(err, "buildkit client")
} }
return &Client{ return &Client{
c: c, c: c,
inputs: map[string]llb.State{}, boot: boot,
bootdir: bootdir,
input: `{}`,
localdirs: map[string]string{}, localdirs: map[string]string{},
}, nil }, nil
} }
func (c *Client) ConnectInput(target string, input interface{}) error { func (c *Client) LocalDirs() ([]string, error) {
var st llb.State boot, err := c.BootScript()
switch in := input.(type) {
case llb.State:
st = in
case string:
// Generate a random local input label for security
st = c.AddLocalDir(in, target)
default:
return fmt.Errorf("unsupported input type")
}
c.inputs[bkInputKey+target] = st
return nil
}
func (c *Client) AddLocalDir(dir, label string, opts ...llb.LocalOption) llb.State {
c.localdirs[label] = dir
return llb.Local(label, opts...)
}
// Set cue config for future calls.
// input can be:
// - llb.State: valid cue config directory
// - io.Reader: valid cue source
// - string: local path to valid cue file or directory
// - func(llb.State)llb.Stte: modify existing state
func (c *Client) SetConfig(inputs ...interface{}) error {
for _, input := range inputs {
if err := c.setConfig(input); err != nil {
return err
}
}
return nil
}
func (c *Client) setConfig(input interface{}) error {
var st llb.State
switch in := input.(type) {
case llb.State:
st = in
case func(llb.State) llb.State:
// Modify previous state
last, ok := c.inputs[bkConfigKey]
if !ok {
last = llb.Scratch()
}
st = in(last)
case io.Reader:
contents, err := ioutil.ReadAll(in)
if err != nil { if err != nil {
return err return nil, err
} }
st = llb.Scratch().File(llb.Mkfile( return boot.LocalDirs()
"config.cue",
0660,
contents,
))
// Interpret string as a path (dir or file)
case string:
info, err := os.Stat(in)
if err != nil {
return err
}
if info.IsDir() {
// FIXME: include pattern *.cue ooh yeah
st = c.AddLocalDir(in, "config",
//llb.IncludePatterns([]string{"*.cue", "cue.mod"})),
llb.FollowPaths([]string{"*.cue", "cue.mod"}),
)
} else {
f, err := os.Open(in)
if err != nil {
return err
}
defer f.Close()
return c.SetConfig(f)
}
}
c.inputs[bkConfigKey] = st
return nil
} }
func (c *Client) Run(ctx context.Context, action string) (*Output, error) { func (c *Client) BootScript() (*Script, error) {
debugf("compiling boot script: %q\n", c.boot)
cc := &Compiler{}
src, err := cc.Compile("boot.cue", c.boot)
if err != nil {
return nil, errors.Wrap(err, "compile")
}
src, err = src.MergeTarget(c.bootdir, "bootdir")
if err != nil {
return nil, err
}
return src.Get("bootscript").Script()
}
func (c *Client) Compute(ctx context.Context) (*Value, error) {
cc := &Compiler{}
out, err := cc.EmptyStruct()
if err != nil {
return nil, err
}
// Spawn Build() goroutine // Spawn Build() goroutine
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
events := make(chan *bk.SolveStatus) events := make(chan *bk.SolveStatus)
outr, outw := io.Pipe() outr, outw := io.Pipe()
// Spawn build function // Spawn build function
eg.Go(c.buildfn(ctx, action, events, outw)) eg.Go(c.buildfn(ctx, events, outw))
// Spawn print function(s) // Spawn print function(s)
dispCtx := context.TODO() dispCtx := context.TODO()
var eventsdup chan *bk.SolveStatus var eventsdup chan *bk.SolveStatus
@ -164,21 +124,36 @@ func (c *Client) Run(ctx context.Context, action string) (*Output, error) {
} }
eg.Go(c.printfn(dispCtx, events, eventsdup)) eg.Go(c.printfn(dispCtx, events, eventsdup))
// Retrieve output // Retrieve output
out := NewOutput()
eg.Go(c.outputfn(ctx, outr, out)) eg.Go(c.outputfn(ctx, outr, out))
return out, eg.Wait() return out, eg.Wait()
} }
func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveStatus, w io.WriteCloser) func() error { func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.WriteCloser) func() error {
return func() error { return func() (err error) {
defer debugf("buildfn complete") defer func() {
debugf("buildfn complete, err=%q", err)
if err != nil {
// Close exporter pipe so that export processor can return
w.Close()
}
}()
boot, err := c.BootScript()
if err != nil {
close(ch)
return errors.Wrap(err, "assemble boot script")
}
bootSource, err := boot.Value().Source()
if err != nil {
close(ch)
return errors.Wrap(err, "serialize boot script")
}
// Setup solve options // Setup solve options
opts := bk.SolveOpt{ opts := bk.SolveOpt{
FrontendAttrs: map[string]string{ FrontendAttrs: map[string]string{
bkActionKey: action, bkInputKey: c.input,
bkBootKey: string(bootSource),
}, },
LocalDirs: c.localdirs, LocalDirs: map[string]string{},
FrontendInputs: c.inputs,
// FIXME: catch output & return as cue value // FIXME: catch output & return as cue value
Exports: []bk.ExportEntry{ Exports: []bk.ExportEntry{
{ {
@ -189,16 +164,18 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt
}, },
}, },
} }
// Setup frontend // Connect local dirs
bkFrontend := c.BKFrontend localdirs, err := c.LocalDirs()
if bkFrontend == nil { if err != nil {
r := &Runtime{} close(ch)
bkFrontend = r.BKFrontend return errors.Wrap(err, "connect local dirs")
} }
resp, err := c.c.Build(ctx, opts, "", bkFrontend, ch) for _, dir := range localdirs {
opts.LocalDirs[dir] = dir
}
// Call buildkit solver
resp, err := c.c.Build(ctx, opts, "", Compute, ch)
if err != nil { if err != nil {
// Close exporter pipe so that export processor can return
w.Close()
err = errors.New(bkCleanError(err.Error())) err = errors.New(bkCleanError(err.Error()))
return errors.Wrap(err, "buildkit solve") return errors.Wrap(err, "buildkit solve")
} }
@ -211,7 +188,7 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt
} }
// Read tar export stream from buildkit Build(), and extract cue output // Read tar export stream from buildkit Build(), and extract cue output
func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func() error { func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Value) func() error {
return func() error { return func() error {
defer debugf("outputfn complete") defer debugf("outputfn complete")
tr := tar.NewReader(r) tr := tar.NewReader(r)
@ -229,14 +206,13 @@ func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func()
continue continue
} }
debugf("outputfn: compiling & merging %q", h.Name) debugf("outputfn: compiling & merging %q", h.Name)
// FIXME: only doing this for debug. you can pass tr directly as io.Reader.
contents, err := ioutil.ReadAll(tr) cc := out.Compiler()
v, err := cc.Compile(h.Name, tr)
if err != nil { if err != nil {
return err return err
} }
//if err := out.FillSource(h.Name, tr); err != nil { if err := out.Fill(v); err != nil {
if err := out.FillSource(h.Name, contents); err != nil {
debugf("error with %s: contents=\n------\n%s\n-----\n", h.Name, contents)
return errors.Wrap(err, h.Name) return errors.Wrap(err, h.Name)
} }
debugf("outputfn: DONE: compiling & merging %q", h.Name) debugf("outputfn: DONE: compiling & merging %q", h.Name)
@ -327,7 +303,7 @@ func (c *Client) printfn(ctx context.Context, ch, ch2 chan *bk.SolveStatus) func
for _, v := range status.Vertexes { for _, v := range status.Vertexes {
p := cue.ParsePath(v.Name) p := cue.ParsePath(v.Name)
if err := p.Err(); err != nil { if err := p.Err(); err != nil {
debugf("ignoring buildkit vertex %q: not a valid cue path", p.String()) debugf("ignoring buildkit vertex %q: not a valid cue path", v.Name)
continue continue
} }
n := &Node{ n := &Node{
@ -377,58 +353,3 @@ func (c *Client) dockerprintfn(ctx context.Context, ch chan *bk.SolveStatus, out
return progressui.DisplaySolveStatus(ctx, "", cons, out, ch) return progressui.DisplaySolveStatus(ctx, "", cons, out, ch)
} }
} }
type Output struct {
r *cue.Runtime
inst *cue.Instance
}
func NewOutput() *Output {
r := &cue.Runtime{}
inst, _ := r.Compile("", "")
return &Output{
r: r,
inst: inst,
}
}
func (o *Output) Print(w io.Writer) error {
v := o.Cue().Value().Eval()
b, err := cueformat.Node(v.Syntax())
if err != nil {
return err
}
_, err = w.Write(b)
return err
}
func (o *Output) JSON() JSON {
return cueToJSON(o.Cue().Value())
}
func (o *Output) Cue() *cue.Instance {
return o.inst
}
func (o *Output) FillSource(filename string, x interface{}) error {
inst, err := o.r.Compile(filename, x)
if err != nil {
return fmt.Errorf("compile %s: %s", filename, cueerrors.Details(err, nil))
}
if err := o.FillValue(inst.Value()); err != nil {
return fmt.Errorf("merge %s: %s", filename, cueerrors.Details(err, nil))
}
return nil
}
func (o *Output) FillValue(x interface{}) error {
inst, err := o.inst.Fill(x)
if err != nil {
return err
}
if err := inst.Value().Validate(); err != nil {
return err
}
o.inst = inst
return nil
}

113
dagger/compiler.go Normal file
View File

@ -0,0 +1,113 @@
//go:generate sh gen.sh
package dagger
import (
"context"
"path"
"path/filepath"
"sync"
"cuelang.org/go/cue"
cueload "cuelang.org/go/cue/load"
"github.com/pkg/errors"
)
// Polyfill for a cue runtime
// (we call it compiler to avoid confusion with dagger runtime)
// Use this instead of cue.Runtime
type Compiler struct {
sync.Mutex
cue.Runtime
spec *Spec
}
func (cc *Compiler) Cue() *cue.Runtime {
return &(cc.Runtime)
}
func (cc *Compiler) Spec() (*Spec, error) {
if cc.spec != nil {
return cc.spec, nil
}
v, err := cc.Compile("spec.cue", DaggerSpec)
if err != nil {
return nil, err
}
spec, err := v.Spec()
if err != nil {
return nil, err
}
cc.spec = spec
return spec, nil
}
// Compile an empty struct
func (cc *Compiler) EmptyStruct() (*Value, error) {
return cc.Compile("", "")
}
func (cc *Compiler) Compile(name string, src interface{}) (*Value, error) {
inst, err := cc.Cue().Compile(name, src)
if err != nil {
// FIXME: cleaner way to unwrap cue error details?
return nil, cueErr(err)
}
return cc.Wrap(inst.Value(), inst), nil
}
func (cc *Compiler) CompileScript(name string, src interface{}) (*Script, error) {
v, err := cc.Compile(name, src)
if err != nil {
return nil, err
}
return v.Script()
}
// Build a cue configuration tree from the files in fs.
func (cc *Compiler) Build(ctx context.Context, fs FS, args ...string) (*Value, error) {
debugf("Compiler.Build")
defer debugf("COMPLETE: Compiler.Build")
// The CUE overlay needs to be prefixed by a non-conflicting path with the
// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
// files it finds locally.
const overlayPrefix = "/config"
buildConfig := &cueload.Config{
Dir: overlayPrefix,
Overlay: map[string]cueload.Source{},
}
buildArgs := args
err := fs.Walk(ctx, func(p string, f Stat) error {
debugf(" Compiler.Build: processing %q", p)
if f.IsDir() {
return nil
}
if filepath.Ext(p) != ".cue" {
return nil
}
contents, err := fs.ReadFile(ctx, p)
if err != nil {
return errors.Wrap(err, p)
}
overlayPath := path.Join(overlayPrefix, p)
buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents)
return nil
})
if err != nil {
return nil, err
}
instances := cueload.Instances(buildArgs, buildConfig)
if len(instances) != 1 {
return nil, errors.New("only one package is supported at a time")
}
inst, err := cc.Cue().Build(instances[0])
if err != nil {
return nil, err
}
return cc.Wrap(inst.Value(), inst), nil
}
func (cc *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value {
return wrapValue(v, inst, cc)
}

66
dagger/component.go Normal file
View File

@ -0,0 +1,66 @@
package dagger
import (
"context"
)
type Component struct {
v *Value
}
func (c *Component) Value() *Value {
return c.v
}
func (c *Component) Exists() bool {
// Does #dagger exist?
if c.Config().Err() != nil {
return false
}
return true
}
// Return the contents of the "#dagger" annotation.
func (c *Component) Config() *Value {
return c.Value().Get("#dagger")
}
// Verify that this component respects the dagger component spec.
//
// NOTE: calling matchSpec("#Component") is not enough because
// it does not match embedded scalars.
func (c Component) Validate() error {
return c.Config().Validate("#ComponentConfig")
}
// Return this component's compute script.
func (c Component) ComputeScript() (*Script, error) {
return c.Value().Get("#dagger.compute").Script()
}
// Compute the configuration for this component.
// Note that we simply execute the underlying compute script from an
// empty filesystem state.
// (It is never correct to pass an input filesystem state to compute a component)
func (c *Component) Compute(ctx context.Context, s Solver, out Fillable) (FS, error) {
return c.Execute(ctx, s.Scratch(), out)
}
// A component implements the Executable interface by returning its
// compute script.
// See Value.Executable().
func (c *Component) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
script, err := c.ComputeScript()
if err != nil {
return fs, err
}
return script.Execute(ctx, fs, out)
}
func (c *Component) Walk(fn func(*Op) error) error {
script, err := c.ComputeScript()
if err != nil {
return err
}
return script.Walk(fn)
}

41
dagger/compute.go Normal file
View File

@ -0,0 +1,41 @@
package dagger
import (
"context"
"fmt"
cueerrors "cuelang.org/go/cue/errors"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
)
// Buildkit compute entrypoint (BK calls if "solve" or "build")
// Use by wrapping in a buildkit client Build call, or buildkit frontend.
func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
// FIXME: wrap errors to avoid crashing buildkit Build()
// with cue error types (why??)
defer func() {
if err != nil {
err = fmt.Errorf("%s", cueerrors.Details(err, nil))
debugf("execute returned an error. Wrapping...")
}
}()
debugf("initializing env")
env, err := NewEnv(ctx, c)
if err != nil {
return nil, err
}
debugf("computing env")
// Compute output overlay
if err := env.Compute(ctx); err != nil {
return nil, err
}
debugf("exporting env")
// Export env to a cue directory
outdir := NewSolver(c).Scratch()
outdir, err = env.Export(outdir)
if err != nil {
return nil, err
}
// Wrap cue directory in buildkit result
return outdir.Result(ctx)
}

186
dagger/env.go Normal file
View File

@ -0,0 +1,186 @@
package dagger
import (
"context"
"os"
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Env struct {
// Base config
base *Value
// Input overlay: user settings, external directories, secrets...
input *Value
// Output overlay: computed values, generated directories
output *Value
// Buildkit solver
s Solver
// shared cue compiler
// (because cue API requires shared runtime for everything)
cc *Compiler
}
// Initialize a new environment
func NewEnv(ctx context.Context, c bkgw.Client) (*Env, error) {
cc := &Compiler{}
// 1. Load base config (specified by client)
debugf("Loading base configuration")
base, err := envBase(ctx, c, cc)
if err != nil {
return nil, err
}
// 2. Load input overlay (specified by client)
debugf("Loading input overlay")
input, err := envInput(ctx, c, cc)
if err != nil {
return nil, err
}
if _, err := base.Merge(input); err != nil {
return nil, errors.Wrap(err, "merge base & input")
}
return &Env{
base: base,
input: input,
s: NewSolver(c),
cc: cc,
}, nil
}
// Compute missing values in env configuration, and write them to state.
func (env *Env) Compute(ctx context.Context) error {
debugf("Computing environment")
output, err := env.Walk(ctx, func(c *Component, out Fillable) error {
_, err := c.Compute(ctx, env.s, out)
return err
})
if err != nil {
return err
}
env.output = output
return nil
}
// Export env to a directory of cue files
// (Use with FS.Change)
func (env *Env) Export(fs FS) (FS, error) {
// FIXME: we serialize as JSON to guarantee a self-contained file.
// Value.Save() leaks imports, so requires a shared cue.mod with
// client which is undesirable.
// Once Value.Save() resolves non-builtin imports with a tree shake,
// we can use it here.
fs = env.base.SaveJSON(fs, "base.cue")
fs = env.input.SaveJSON(fs, "input.cue")
if env.output != nil {
fs = env.output.SaveJSON(fs, "output.cue")
}
return fs, nil
}
type EnvWalkFunc func(*Component, Fillable) error
// Walk components and return any computed values
func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
debugf("Env.Walk")
defer debugf("COMPLETE: Env.Walk")
// Cueflow cue instance
// FIXME: make this cleaner in *Value by keeping intermediary instances
flowInst, err := env.base.CueInst().Fill(env.input.CueInst())
if err != nil {
return nil, err
}
// Initialize empty output
out, err := env.cc.EmptyStruct()
if err != nil {
return nil, err
}
// Cueflow config
flowCfg := &cueflow.Config{
UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error {
debugf("compute step")
if t == nil {
return nil
}
debugf("cueflow task %q: %s", t.Path().String(), t.State().String())
if t.State() != cueflow.Terminated {
return nil
}
debugf("cueflow task %q: filling result", t.Path().String())
// Merge task value into output
var err error
// FIXME: does cueflow.Task.Value() contain only filled values,
// or base + filled?
out, err = out.MergePath(t.Value(), t.Path())
if err != nil {
return err
}
return nil
},
}
// Cueflow match func
flowMatchFn := func(v cue.Value) (cueflow.Runner, error) {
val := env.cc.Wrap(v, flowInst)
c, err := val.Component()
if os.IsNotExist(err) {
// Not a component: skip
return nil, nil
}
if err != nil {
return nil, err
}
return cueflow.RunnerFunc(func(t *cueflow.Task) error {
return fn(c, t)
}), nil
}
// Orchestrate execution with cueflow
flow := cueflow.New(flowCfg, flowInst, flowMatchFn)
if err := flow.Run(ctx); err != nil {
return out, err
}
return out, nil
}
func envBase(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
// 1. Receive boot script from client.
debugf("retrieving boot script")
bootSrc, exists := c.BuildOpts().Opts["boot"]
if !exists {
// No boot script: return empty base config
return cc.EmptyStruct()
}
// 2. Compile & execute boot script
debugf("compiling boot script")
boot, err := cc.CompileScript("boot.cue", bootSrc)
if err != nil {
return nil, errors.Wrap(err, "compile boot script")
}
debugf("executing boot script")
bootState, err := boot.Execute(ctx, NewSolver(c).Scratch(), Discard())
if err != nil {
return nil, errors.Wrap(err, "execute boot script")
}
// 3. load cue files produced by bootstrap script
// FIXME: BuildAll() to force all files (no required package..)
debugf("building cue configuration from boot state")
base, err := cc.Build(ctx, bootState)
debugf("done building cue configuration: err=%q", err)
return base, err
}
func envInput(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
// 1. Receive input overlay from client.
// This is used to provide run-time settings, directories..
inputSrc, exists := c.BuildOpts().Opts["input"]
if !exists {
// No input overlay: return empty tree
return cc.EmptyStruct()
}
return cc.Compile("input.cue", inputSrc)
}

143
dagger/fs.go Normal file
View File

@ -0,0 +1,143 @@
package dagger
import (
"context"
"path"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
fstypes "github.com/tonistiigi/fsutil/types"
)
type Stat struct {
*fstypes.Stat
}
type FS struct {
// Before last solve
input llb.State
// After last solve
output bkgw.Reference
// How to produce the output
s Solver
}
func (fs FS) Solver() Solver {
return fs.s
}
// Compute output from input, if not done already.
// This method uses a pointer receiver to simplify
// calling it, since it is called in almost every
// other method.
func (fs *FS) solve(ctx context.Context) error {
if fs.output != nil {
return nil
}
output, err := fs.s.Solve(ctx, fs.input)
if err != nil {
return err
}
fs.output = output
return nil
}
func (fs FS) ReadFile(ctx context.Context, filename string) ([]byte, error) {
// Lazy solve
if err := (&fs).solve(ctx); err != nil {
return nil, err
}
return fs.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename})
}
func (fs FS) ReadDir(ctx context.Context, dir string) ([]Stat, error) {
// Lazy solve
if err := (&fs).solve(ctx); err != nil {
return nil, err
}
st, err := fs.output.ReadDir(ctx, bkgw.ReadDirRequest{
Path: dir,
})
if err != nil {
return nil, err
}
out := make([]Stat, len(st))
for i := range st {
out[i] = Stat{
Stat: st[i],
}
}
return out, nil
}
func (fs FS) walk(ctx context.Context, p string, fn WalkFunc) error {
files, err := fs.ReadDir(ctx, p)
if err != nil {
return err
}
for _, f := range files {
fPath := path.Join(p, f.GetPath())
if err := fn(fPath, f); err != nil {
return err
}
if f.IsDir() {
if err := fs.walk(ctx, fPath, fn); err != nil {
return err
}
}
}
return nil
}
type WalkFunc func(string, Stat) error
func (fs FS) Walk(ctx context.Context, fn WalkFunc) error {
// Lazy solve
if err := (&fs).solve(ctx); err != nil {
return err
}
return fs.walk(ctx, "/", fn)
}
type ChangeFunc func(llb.State) llb.State
func (fs FS) Change(changes ...ChangeFunc) FS {
for _, change := range changes {
fs = fs.Set(change(fs.input))
}
return fs
}
func (fs FS) Set(st llb.State) FS {
fs.input = st
fs.output = nil
return fs
}
func (fs FS) Solve(ctx context.Context) (FS, error) {
if err := (&fs).solve(ctx); err != nil {
return fs, err
}
return fs, nil
}
func (fs FS) LLB() llb.State {
return fs.input
}
func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) {
if err := (&fs).solve(ctx); err != nil {
return nil, err
}
return fs.output, nil
}
func (fs FS) Result(ctx context.Context) (*bkgw.Result, error) {
res := bkgw.NewResult()
ref, err := fs.Ref(ctx)
if err != nil {
return nil, err
}
res.SetRef(ref)
return res, nil
}

30
dagger/gen.go
View File

@ -46,6 +46,7 @@ package dagger
// The contents of a #dagger annotation // The contents of a #dagger annotation
#ComponentConfig: { #ComponentConfig: {
// FIXME: deprecated
input?: bool input?: bool
// script to compute the value // script to compute the value
@ -91,7 +92,7 @@ package dagger
#Script: [...#Op] #Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy
// Export a value from fs state to cue // Export a value from fs state to cue
#Export: { #Export: {
@ -101,22 +102,24 @@ package dagger
format: "json"|"yaml"|*"string"|"number"|"boolean" format: "json"|"yaml"|*"string"|"number"|"boolean"
} }
#Load: #LoadComponent| #LoadScript #Local: {
#LoadComponent: { do: "local"
do: "load" dir: string
from: #Component include: [...string] | *[]
}
#LoadScript: {
do: "load"
from: #Script
} }
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Exec: { #Exec: {
do: "exec" do: "exec"
args: [...string] args: [...string]
env: [string]: string env?: [string]: string
always: true | *false always?: true | *false
dir: string | *"/" dir: string | *"/"
mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
} }
@ -146,11 +149,10 @@ package dagger
#Copy: { #Copy: {
do: "copy" do: "copy"
from: #Script | #Component from: #Script | #Component
src: string | *"/" src?: string | *"/"
dest: string | *"/" dest?: string | *"/"
} }
#TestScript: #Script & [ #TestScript: #Script & [
{ do: "fetch-container", ref: "alpine:latest" }, { do: "fetch-container", ref: "alpine:latest" },
{ do: "exec", args: ["echo", "hello", "world" ] } { do: "exec", args: ["echo", "hello", "world" ] }

326
dagger/job.go
View File

@ -1,326 +0,0 @@
package dagger
import (
"context"
"fmt"
"path"
"path/filepath"
"strings"
"cuelang.org/go/cue"
cueerrors "cuelang.org/go/cue/errors"
cueload "cuelang.org/go/cue/load"
cueflow "cuelang.org/go/tools/flow"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Solver interface {
Solve(context.Context, llb.State) (bkgw.Reference, error)
}
// 1 buildkit build = 1 job
type Job struct {
c bkgw.Client
// needed for cue operations
r *Runtime
}
// Execute and wrap the result in a buildkit result
func (job Job) BKExecute(ctx context.Context) (_r *bkgw.Result, _e error) {
debugf("Executing bk frontend")
// wrap errors to avoid crashing buildkit with cue error types (why??)
defer func() {
if _e != nil {
_e = fmt.Errorf("%s", cueerrors.Details(_e, nil))
debugf("execute returned an error. Wrapping...")
}
}()
out, err := job.Execute(ctx)
if err != nil {
return nil, err
}
// encode job output to buildkit result
debugf("[runtime] job executed. Encoding output")
// FIXME: we can't serialize result to standalone cue (with no imports).
// So the client cannot safely compile output without access to the same cue.mod
// as the runtime (which we don't want).
// So for now we return the output as json, still parsed as cue on the client
// to keep our options open. Once there is a "tree shake" primitive, we can
// use that to return cue.
//
// Uncomment to return actual cue:
// ----
// outbytes, err := cueformat.Node(out.Value().Eval().Syntax())
// if err != nil {
// return nil, err
// }
// ----
outbytes := cueToJSON(out.Value())
debugf("[runtime] output encoded. Writing output to exporter")
outref, err := job.Solve(ctx,
llb.Scratch().File(llb.Mkfile("computed.cue", 0600, outbytes)),
)
if err != nil {
return nil, err
}
debugf("[runtime] output written to exporter. returning to buildkit solver")
res := bkgw.NewResult()
res.SetRef(outref)
return res, nil
}
func (job Job) Execute(ctx context.Context) (_i *cue.Instance, _e error) {
debugf("[runtime] Execute()")
defer func() { debugf("[runtime] DONE Execute(): err=%v", _e) }()
state, err := job.Config(ctx)
if err != nil {
return nil, err
}
// Merge input information into the cue config
inputs, err := job.Inputs(ctx)
if err != nil {
return nil, err
}
for target := range inputs {
// FIXME: cleaner code generation, missing cue.Value.FillPath
state, err = job.r.fill(state, `#dagger: input: true`, target)
if err != nil {
return nil, errors.Wrapf(err, "connect input %q", target)
}
}
action := job.Action()
switch action {
case "compute":
return job.doCompute(ctx, state)
case "export":
return job.doExport(ctx, state)
default:
return job.doExport(ctx, state)
}
}
func (job Job) doExport(ctx context.Context, state *cue.Instance) (*cue.Instance, error) {
return state, nil
}
func (job Job) doCompute(ctx context.Context, state *cue.Instance) (*cue.Instance, error) {
out, err := job.r.Compile("computed.cue", "")
if err != nil {
return nil, err
}
// Setup cueflow
debugf("Setting up cueflow")
flow := cueflow.New(
&cueflow.Config{
UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error {
debugf("cueflow event")
if t == nil {
return nil
}
debugf("cueflow task %q: %s", t.Path().String(), t.State().String())
if t.State() == cueflow.Terminated {
debugf("cueflow task %q: filling result", t.Path().String())
out, err = out.Fill(t.Value(), cuePathToStrings(t.Path())...)
if err != nil {
return err
}
// FIXME: catch merge errors early with state
}
return nil
},
},
state,
// Task match func
func(v cue.Value) (cueflow.Runner, error) {
// Is v a component (has #dagger) with a field 'compute' ?
isComponent, err := job.r.isComponent(v, "compute")
if err != nil {
return nil, err
}
if !isComponent {
return nil, nil
}
debugf("[%s] component detected\n", v.Path().String())
// task runner func
runner := cueflow.RunnerFunc(func(t *cueflow.Task) error {
computeScript := t.Value().LookupPath(cue.ParsePath("#dagger.compute"))
script, err := job.newScript(computeScript)
if err != nil {
return err
}
// Run the script & fill the result into the task
return script.Run(ctx, t)
})
return runner, nil
},
)
debugf("Running cueflow")
if err := flow.Run(ctx); err != nil {
return nil, err
}
debugf("Completed cueflow run. Merging result.")
state, err = state.Fill(out)
if err != nil {
return nil, err
}
debugf("Result merged")
// Return only the computed values
return out, nil
}
func (job Job) bk() bkgw.Client {
return job.c
}
func (job Job) Action() string {
opts := job.bk().BuildOpts().Opts
if action, ok := opts[bkActionKey]; ok {
return action
}
return ""
}
// Load the cue config for this job
// (received as llb input)
func (job Job) Config(ctx context.Context) (*cue.Instance, error) {
src := llb.Local(bkConfigKey,
llb.SessionID(job.bk().BuildOpts().SessionID),
llb.SharedKeyHint(bkConfigKey),
llb.WithCustomName("load config"),
)
bkInputs, err := job.bk().Inputs(ctx)
if err != nil {
return nil, err
}
if st, ok := bkInputs[bkConfigKey]; ok {
src = st
}
// job.runDebug(ctx, src, "ls", "-la", "/mnt")
return job.LoadCue(ctx, src)
}
func (job Job) runDebug(ctx context.Context, mnt llb.State, args ...string) error {
opts := []llb.RunOption{
llb.Args(args),
llb.AddMount("/mnt", mnt),
}
cmd := llb.Image("alpine").Run(opts...).Root()
ref, err := job.Solve(ctx, cmd)
if err != nil {
return errors.Wrap(err, "debug")
}
// force non-lazy solve
if _, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{Path: "/"}); err != nil {
return errors.Wrap(err, "debug")
}
return nil
}
func (job Job) Inputs(ctx context.Context) (map[string]llb.State, error) {
bkInputs, err := job.bk().Inputs(ctx)
if err != nil {
return nil, err
}
inputs := map[string]llb.State{}
for key, input := range bkInputs {
if !strings.HasPrefix(key, bkInputKey) {
continue
}
target := strings.Replace(key, bkInputKey, "", 1)
targetPath := cue.ParsePath(target)
if err := targetPath.Err(); err != nil {
return nil, errors.Wrapf(err, "input target %q", target)
}
// FIXME: check that the path can be passed to Fill
// (eg. only regular fields, no array indexes, no defs)
// see cuePathToStrings
inputs[target] = input
}
return inputs, nil
}
// loadFiles recursively loads all .cue files from a buildkit gateway
// FIXME: this is highly inefficient.
func loadFiles(ctx context.Context, ref bkgw.Reference, p, overlayPrefix string, overlay map[string]cueload.Source) error {
// FIXME: we cannot use `IncludePattern` here, otherwise sub directories
// (e.g. "cue.mod") will be skipped.
files, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{
Path: p,
})
if err != nil {
return err
}
for _, f := range files {
fPath := path.Join(p, f.GetPath())
if f.IsDir() {
if err := loadFiles(ctx, ref, fPath, overlayPrefix, overlay); err != nil {
return err
}
continue
}
if filepath.Ext(fPath) != ".cue" {
continue
}
contents, err := ref.ReadFile(ctx, bkgw.ReadRequest{
Filename: fPath,
})
if err != nil {
return errors.Wrap(err, f.GetPath())
}
overlay[path.Join(overlayPrefix, fPath)] = cueload.FromBytes(contents)
}
return nil
}
func (job Job) LoadCue(ctx context.Context, st llb.State, args ...string) (*cue.Instance, error) {
// The CUE overlay needs to be prefixed by a non-conflicting path with the
// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
// files it finds locally.
const overlayPrefix = "/config"
buildConfig := &cueload.Config{
Dir: overlayPrefix,
Overlay: map[string]cueload.Source{},
}
buildArgs := args
// Inject cue files from llb state into overlay
ref, err := job.Solve(ctx, st)
if err != nil {
return nil, err
}
if err := loadFiles(ctx, ref, ".", overlayPrefix, buildConfig.Overlay); err != nil {
return nil, err
}
instances := cueload.Instances(buildArgs, buildConfig)
if len(instances) != 1 {
return nil, errors.New("only one package is supported at a time")
}
inst, err := job.r.Build(instances[0])
if err != nil {
return nil, cueErr(err)
}
return inst, nil
}
func (job Job) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
// marshal llb
def, err := st.Marshal(ctx, llb.LinuxAmd64)
if err != nil {
return nil, err
}
// call solve
res, err := job.bk().Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()})
if err != nil {
return nil, err
}
// always use single reference (ignore multiple outputs & metadata)
return res.SingleRef()
}

38
dagger/mount.go Normal file
View File

@ -0,0 +1,38 @@
package dagger
import (
"context"
"fmt"
"github.com/moby/buildkit/client/llb"
"github.com/pkg/errors"
)
type Mount struct {
dest string
v *Value
}
func (mnt *Mount) Validate(defs ...string) error {
return mnt.v.Validate(append(defs, "#Mount")...)
}
func (mnt *Mount) LLB(ctx context.Context, s Solver) (llb.RunOption, error) {
if err := mnt.Validate("#MountTmp"); err == nil {
return llb.AddMount(mnt.dest, llb.Scratch(), llb.Tmpfs()), nil
}
if err := mnt.Validate("#MountCache"); err == nil {
// FIXME: cache mount
return nil, fmt.Errorf("FIXME: cache mount not yet implemented")
}
// Compute source component or script, discarding fs writes & output value
from, err := mnt.v.Lookup("from").Executable()
if err != nil {
return nil, errors.Wrap(err, "from")
}
fromFS, err := from.Execute(ctx, s.Scratch(), Discard())
if err != nil {
return nil, err
}
return llb.AddMount(mnt.dest, fromFS.LLB()), nil
}

256
dagger/op.go Normal file
View File

@ -0,0 +1,256 @@
package dagger
import (
"context"
"encoding/json"
"fmt"
"github.com/moby/buildkit/client/llb"
"github.com/pkg/errors"
)
type Op struct {
v *Value
}
func (op *Op) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
action, err := op.Action()
if err != nil {
return fs, err
}
return action(ctx, fs, out)
}
func (op *Op) Walk(fn func(*Op) error) error {
debugf("Walk %v", op.v)
isCopy := (op.Validate("#Copy") == nil)
isLoad := (op.Validate("#Load") == nil)
if isCopy || isLoad {
debugf("MATCH %v", op.v)
if from, err := op.Get("from").Executable(); err == nil {
if err := from.Walk(fn); err != nil {
return err
}
}
// FIXME: we tolerate "from" which is not executable
}
if err := op.Validate("#Exec"); err == nil {
return op.Get("mount").RangeStruct(func(k string, v *Value) error {
if from, err := op.Get("from").Executable(); err == nil {
if err := from.Walk(fn); err != nil {
return err
}
}
return nil
})
}
// depth first
return fn(op)
}
type Action func(context.Context, FS, Fillable) (FS, error)
func (op *Op) Action() (Action, error) {
// An empty struct is allowed as a no-op, to be
// more tolerant of if() in arrays.
if op.v.IsEmptyStruct() {
return op.Nothing, nil
}
// Manually check for a 'do' field.
// This is necessary because Runtime.ValidateSpec has a bug
// where an empty struct matches everything.
// see https://github.com/cuelang/cue/issues/566#issuecomment-749735878
// Once the bug is fixed, the manual check can be removed.
if _, err := op.Get("do").String(); err != nil {
return nil, fmt.Errorf("invalid action: no \"do\" field")
}
actions := map[string]Action{
"#Copy": op.Copy,
"#Exec": op.Exec,
"#Export": op.Export,
"#FetchContainer": op.FetchContainer,
"#FetchGit": op.FetchGit,
"#Local": op.Local,
}
for def, action := range actions {
if err := op.Validate(def); err == nil {
op.v.Debugf("OP MATCH: %s", def)
return action, nil
}
}
return nil, fmt.Errorf("invalid operation: %s", op.v.JSON())
}
func (op *Op) Validate(defs ...string) error {
defs = append(defs, "#Op")
return op.v.Validate(defs...)
}
func (op *Op) Copy(ctx context.Context, fs FS, out Fillable) (FS, error) {
// Decode copy options
src, err := op.Get("src").String()
if err != nil {
return fs, err
}
dest, err := op.Get("dest").String()
if err != nil {
return fs, err
}
from, err := op.Get("from").Executable()
if err != nil {
return fs, errors.Wrap(err, "from")
}
// Compute source component or script, discarding fs writes & output value
fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard())
if err != nil {
return fs, err
}
return fs.Change(func(st llb.State) llb.State {
return st.File(llb.Copy(
fromFS.LLB(),
src,
dest,
// FIXME: allow more configurable llb options
// For now we define the following convenience presets:
&llb.CopyInfo{
CopyDirContentsOnly: true,
CreateDestPath: true,
AllowWildcard: true,
},
))
}), nil // lazy solve
}
func (op *Op) Nothing(ctx context.Context, fs FS, out Fillable) (FS, error) {
return fs, nil
}
func (op *Op) Local(ctx context.Context, fs FS, out Fillable) (FS, error) {
dir, err := op.Get("dir").String()
if err != nil {
return fs, err
}
var include []string
if err := op.Get("include").Decode(&include); err != nil {
return fs, err
}
return fs.Set(llb.Local(dir, llb.FollowPaths(include))), nil // lazy solve
}
func (op *Op) Exec(ctx context.Context, fs FS, out Fillable) (FS, error) {
var opts []llb.RunOption
var cmd struct {
Args []string
Env map[string]string
Dir string
Always bool
}
op.v.Decode(&cmd)
// marker for status events
// FIXME
opts = append(opts, llb.WithCustomName(op.v.Path().String()))
// args
opts = append(opts, llb.Args(cmd.Args))
// dir
dir := cmd.Dir
if dir == "" {
dir = "/"
}
// env
for k, v := range cmd.Env {
opts = append(opts, llb.AddEnv(k, v))
}
// always?
if cmd.Always {
cacheBuster, err := randomID(8)
if err != nil {
return fs, err
}
opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster))
}
// mounts
if err := op.v.RangeStruct(func(k string, v *Value) error {
mnt, err := v.Mount(k)
if err != nil {
return err
}
opt, err := mnt.LLB(ctx, fs.Solver())
if err != nil {
return err
}
opts = append(opts, opt)
return nil
}); err != nil {
return fs, err
}
// --> Execute
return fs.Change(func(st llb.State) llb.State {
return st.Run(opts...).Root()
}), nil // lazy solve
}
func (op *Op) Export(ctx context.Context, fs FS, out Fillable) (FS, error) {
source, err := op.Get("source").String()
if err != nil {
return fs, err
}
format, err := op.Get("format").String()
if err != nil {
return fs, err
}
contents, err := fs.ReadFile(ctx, source)
if err != nil {
return fs, errors.Wrapf(err, "export %s", source)
}
switch format {
case "string":
if err := out.Fill(string(contents)); err != nil {
return fs, err
}
case "json":
var o interface{}
if err := json.Unmarshal(contents, &o); err != nil {
return fs, err
}
if err := out.Fill(o); err != nil {
return fs, err
}
default:
return fs, fmt.Errorf("unsupported export format: %q", format)
}
return fs, nil
}
func (op *Op) Load(ctx context.Context, fs FS, out Fillable) (FS, error) {
from, err := op.Get("from").Executable()
if err != nil {
return fs, errors.Wrap(err, "load")
}
fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard())
if err != nil {
return fs, errors.Wrap(err, "load: compute source")
}
return fs.Set(fromFS.LLB()), nil
}
func (op *Op) FetchContainer(ctx context.Context, fs FS, out Fillable) (FS, error) {
ref, err := op.Get("ref").String()
if err != nil {
return fs, err
}
return fs.Set(llb.Image(ref)), nil
}
func (op *Op) FetchGit(ctx context.Context, fs FS, out Fillable) (FS, error) {
remote, err := op.Get("remote").String()
if err != nil {
return fs, err
}
ref, err := op.Get("ref").String()
if err != nil {
return fs, err
}
return fs.Set(llb.Git(remote, ref)), nil // lazy solve
}
func (op *Op) Get(target string) *Value {
return op.v.Get(target)
}

32
dagger/op_test.go Normal file
View File

@ -0,0 +1,32 @@
package dagger
import (
"testing"
)
func TestCopyMatch(t *testing.T) {
cc := &Compiler{}
src := `do: "copy", from: [{do: "local", dir: "foo"}]`
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
op, err := v.Op()
if err != nil {
t.Fatal(err)
}
if err := op.Validate("#Copy"); err != nil {
t.Fatal(err)
}
n := 0
err = op.Walk(func(op *Op) error {
n += 1
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatal(n)
}
}

109
dagger/runtime.go
View File

@ -1,109 +0,0 @@
//go:generate sh gen.sh
package dagger
import (
"context"
"fmt"
"sync"
"cuelang.org/go/cue"
cueerrors "cuelang.org/go/cue/errors"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Runtime struct {
l sync.Mutex
cue.Runtime
}
func (r *Runtime) Cue() *cue.Runtime {
return &(r.Runtime)
}
func (r *Runtime) fill(inst *cue.Instance, v interface{}, target string) (*cue.Instance, error) {
targetPath := cue.ParsePath(target)
if err := targetPath.Err(); err != nil {
return nil, err
}
p := cuePathToStrings(targetPath)
if src, ok := v.(string); ok {
vinst, err := r.Compile(target, src)
if err != nil {
return nil, err
}
return inst.Fill(vinst.Value(), p...)
}
return inst.Fill(v, p...)
}
// func (r Runtime) Run(...)
// Buildkit run entrypoint
func (r *Runtime) BKFrontend(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
return r.newJob(c).BKExecute(ctx)
}
func (r *Runtime) newJob(c bkgw.Client) Job {
return Job{
r: r,
c: c,
}
}
// Check whether a value is a valid component
// FIXME: calling matchSpec("#Component") is not enough because
// it does not match embedded scalars.
func (r *Runtime) isComponent(v cue.Value, fields ...string) (bool, error) {
cfg := v.LookupPath(cue.ParsePath("#dagger"))
if cfg.Err() != nil {
// No "#dagger" -> not a component
return false, nil
}
for _, field := range fields {
if cfg.Lookup(field).Err() != nil {
return false, nil
}
}
if err := r.validateSpec(cfg, "#ComponentConfig"); err != nil {
return true, errors.Wrap(err, "invalid #dagger")
}
return true, nil
}
// eg. validateSpec(op, "#Op")
// eg. validateSpec(dag, "#DAG")
func (r *Runtime) validateSpec(v cue.Value, defpath string) (err error) {
// Expand cue errors to get full details
// FIXME: there is probably a cleaner way to do this.
defer func() {
if err != nil {
err = fmt.Errorf("%s", cueerrors.Details(err, nil))
}
}()
r.l.Lock()
defer r.l.Unlock()
// FIXME cache spec instance
spec, err := r.Compile("dagger.cue", DaggerSpec)
if err != nil {
panic("invalid spec")
}
def := spec.Value().LookupPath(cue.ParsePath(defpath))
if err := def.Err(); err != nil {
return err
}
v = v.Eval()
if err := v.Validate(); err != nil {
return err
}
res := def.Unify(v)
if err := res.Validate(cue.Final()); err != nil {
return err
}
return nil
}
func (r *Runtime) matchSpec(v cue.Value, def string) bool {
return r.validateSpec(v, def) == nil
}

350
dagger/script.go
View File

@ -2,325 +2,69 @@ package dagger
import ( import (
"context" "context"
"encoding/json"
"fmt"
"cuelang.org/go/cue"
"github.com/moby/buildkit/client/llb"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
type Script struct { type Script struct {
v cue.Value v *Value
job Job
// current state
state *State
} }
func (job Job) newScript(v cue.Value) (*Script, error) { func (s Script) Validate() error {
s := &Script{ return s.Value().Validate("#Script")
v: v,
job: job,
state: NewState(job),
}
if err := s.Validate(); err != nil {
return nil, s.err(err, "invalid script")
}
return s, nil
} }
type Action func(context.Context, cue.Value, Fillable) error func (s *Script) Value() *Value {
func (s *Script) Run(ctx context.Context, out Fillable) error {
op, err := s.Cue().List()
if err != nil {
return s.err(err, "run")
}
i := 0
for op.Next() {
// If op is not concrete, interrupt execution without error.
// This allows gradual resolution: compute what you can compute.. leave the rest incomplete.
if !cueIsConcrete(op.Value()) {
debugf("%s: non-concrete op. Leaving script unfinished", op.Value().Path().String())
return nil
}
if err := s.Do(ctx, op.Value(), out); err != nil {
return s.err(err, "run op %d", i+1)
}
i += 1
}
return nil
}
func (s *Script) Do(ctx context.Context, op cue.Value, out Fillable) error {
// Skip no-ops without error (allows more flexible use of if())
// FIXME: maybe not needed once a clear pattern is established for
// how to use if() in a script?
if cueIsEmptyStruct(op) {
return nil
}
actions := map[string]Action{
// "#Copy": s.copy,
"#Exec": s.exec,
"#Export": s.export,
"#FetchContainer": s.fetchContainer,
"#FetchGit": s.fetchGit,
"#Load": s.load,
"#Copy": s.copy,
}
for def, action := range actions {
if s.matchSpec(op, def) {
debugf("OP MATCH: %s: %s: %v", def, op.Path().String(), op)
return action(ctx, op, out)
}
}
return fmt.Errorf("[%s] invalid operation: %s", s.v.Path().String(), cueToJSON(op))
}
func (s *Script) copy(ctx context.Context, v cue.Value, out Fillable) error {
// Decode copy options
var op struct {
Src string
Dest string
}
if err := v.Decode(&op); err != nil {
return err
}
from := v.Lookup("from")
if isComponent, err := s.job.r.isComponent(from); err != nil {
return err
} else if isComponent {
return s.copyComponent(ctx, from, op.Src, op.Dest)
}
if s.matchSpec(from, "#Script") {
return s.copyScript(ctx, from, op.Src, op.Dest)
}
return fmt.Errorf("copy: invalid source")
}
func (s *Script) copyScript(ctx context.Context, from cue.Value, src, dest string) error {
// Load source script
fromScript, err := s.job.newScript(from)
if err != nil {
return err
}
// Execute source script
if err := fromScript.Run(ctx, Discard()); err != nil {
return err
}
return s.State().Change(ctx, func(st llb.State) llb.State {
return st.File(llb.Copy(
fromScript.State().LLB(),
src,
dest,
// FIXME: allow more configurable llb options
// For now we define the following convenience presets:
&llb.CopyInfo{
CopyDirContentsOnly: true,
CreateDestPath: true,
AllowWildcard: true,
},
))
})
}
func (s *Script) copyComponent(ctx context.Context, from cue.Value, src, dest string) error {
return s.copyScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute")), src, dest)
}
func (s *Script) load(ctx context.Context, op cue.Value, out Fillable) error {
from := op.Lookup("from")
isComponent, err := s.job.r.isComponent(from)
if err != nil {
return err
}
if isComponent {
debugf("LOAD: from is a component")
return s.loadScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute")))
}
if s.matchSpec(from, "#Script") {
return s.loadScript(ctx, from)
}
return fmt.Errorf("load: invalid source")
}
func (s *Script) loadScript(ctx context.Context, v cue.Value) error {
from, err := s.job.newScript(v)
if err != nil {
return errors.Wrap(err, "load")
}
// NOTE we discard cue outputs from running the loaded script.
// This means we load the LLB state but NOT the cue exports.
// In other words: cue exports are always private to their original location.
if err := from.Run(ctx, Discard()); err != nil {
return errors.Wrap(err, "load/execute")
}
// overwrite buildkit state from loaded from
s.state = from.state
return nil
}
func (s *Script) exec(ctx context.Context, v cue.Value, out Fillable) error {
var opts []llb.RunOption
var cmd struct {
Args []string
Env map[string]string
Dir string
Always bool
}
v.Decode(&cmd)
// marker for status events
opts = append(opts, llb.WithCustomName(v.Path().String()))
// args
opts = append(opts, llb.Args(cmd.Args))
// dir
dir := cmd.Dir
if dir == "" {
dir = "/"
}
// env
for k, v := range cmd.Env {
opts = append(opts, llb.AddEnv(k, v))
}
// always?
if cmd.Always {
cacheBuster, err := randomID(8)
if err != nil {
return err
}
opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster))
}
// mounts
mnt, _ := v.Lookup("mount").Fields()
for mnt.Next() {
opt, err := s.mount(ctx, mnt.Label(), mnt.Value())
if err != nil {
return err
}
opts = append(opts, opt)
}
// --> Execute
return s.State().Change(ctx, func(st llb.State) llb.State {
return st.Run(opts...).Root()
})
}
func (s *Script) mount(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
if s.matchSpec(source, "#MountTmp") {
return llb.AddMount(dest, llb.Scratch(), llb.Tmpfs()), nil
}
if s.matchSpec(source, "#MountCache") {
// FIXME: cache mount
return nil, fmt.Errorf("FIXME: cache mount not yet implemented")
}
if s.matchSpec(source, "#MountScript") {
return s.mountScript(ctx, dest, source)
}
if s.matchSpec(source, "#MountComponent") {
return s.mountComponent(ctx, dest, source)
}
return nil, fmt.Errorf("mount %s to %s: invalid source", source.Path().String(), dest)
}
// mount when the input is a script (see mountComponent, mountTmpfs, mountCache)
func (s *Script) mountScript(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
script, err := s.job.newScript(source)
if err != nil {
return nil, err
}
// FIXME: this is where we re-run everything,
// and rely on solver cache / dedup
if err := script.Run(ctx, Discard()); err != nil {
return nil, err
}
return llb.AddMount(dest, script.State().LLB()), nil
}
func (s *Script) mountComponent(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
return s.mountScript(ctx, dest, source.LookupPath(cue.ParsePath("from.#dagger.compute")))
}
func (s *Script) fetchContainer(ctx context.Context, v cue.Value, out Fillable) error {
var op struct {
Ref string
}
if err := v.Decode(&op); err != nil {
return errors.Wrap(err, "decode fetch-container")
}
return s.State().Change(ctx, llb.Image(op.Ref))
}
func (s *Script) fetchGit(ctx context.Context, v cue.Value, out Fillable) error {
// See #FetchGit in spec.cue
var op struct {
Remote string
Ref string
}
if err := v.Decode(&op); err != nil {
return errors.Wrap(err, "decode fetch-git")
}
return s.State().Change(ctx, llb.Git(op.Remote, op.Ref))
}
func (s *Script) export(ctx context.Context, v cue.Value, out Fillable) error {
// See #Export in spec.cue
var op struct {
Source string
// FIXME: target
// Target string
Format string
}
v.Decode(&op)
b, err := s.State().ReadFile(ctx, op.Source)
if err != nil {
return err
}
switch op.Format {
case "string":
return out.Fill(string(b))
case "json":
var o interface{}
if err := json.Unmarshal(b, &o); err != nil {
return err
}
return out.Fill(o)
default:
return fmt.Errorf("unsupported export format: %q", op.Format)
}
}
func (s *Script) Cue() cue.Value {
return s.v return s.v
} }
func (s *Script) Location() string { // Run a dagger script
return s.Cue().Path().String() func (s *Script) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
err := s.v.RangeList(func(idx int, v *Value) error {
// If op not concrete, interrupt without error.
// This allows gradual resolution:
// compute what you can compute.. leave the rest incomplete.
if !v.IsConcreteR() {
return nil
}
op, err := v.Op()
if err != nil {
return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len())
}
fs, err = op.Execute(ctx, fs, out)
if err != nil {
return errors.Wrapf(err, "execute op %d/%d", idx+1, s.v.Len())
}
return nil
})
return fs, err
} }
func (s *Script) err(e error, msg string, args ...interface{}) error { func (s *Script) Walk(fn func(op *Op) error) error {
return errors.Wrapf(e, s.Location()+": "+msg, args...) return s.v.RangeList(func(idx int, v *Value) error {
op, err := v.Op()
if err != nil {
return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len())
}
if err := op.Walk(fn); err != nil {
return err
}
return nil
})
} }
func (s *Script) Validate() error { func (s *Script) LocalDirs() ([]string, error) {
return s.job.r.validateSpec(s.Cue(), "#Script") var dirs []string
err := s.Walk(func(op *Op) error {
if err := op.Validate("#Local"); err != nil {
return nil
} }
dir, err := op.Get("dir").String()
func (s *Script) State() *State { if err != nil {
return s.state return err
} }
dirs = append(dirs, dir)
func (s *Script) matchSpec(v cue.Value, def string) bool { return nil
// FIXME: we manually filter out empty structs to avoid false positives })
// This is necessary because Runtime.ValidateSpec has a bug return dirs, err
// where an empty struct matches everything.
// see https://github.com/cuelang/cue/issues/566#issuecomment-749735878
// Once the bug is fixed, the manual check can be removed.
if st, err := v.Struct(); err == nil {
if st.Len() == 0 {
debugf("FIXME: manually filtering out empty struct from spec match")
return false
}
}
return s.job.r.matchSpec(v, def)
} }

78
dagger/script_test.go Normal file
View File

@ -0,0 +1,78 @@
package dagger
import (
"strings"
"testing"
)
func TestWalkBootScript(t *testing.T) {
cc := &Compiler{}
script, err := cc.CompileScript("boot.cue", defaultBootScript)
if err != nil {
t.Fatal(err)
}
dirs, err := script.LocalDirs()
if err != nil {
t.Fatal(err)
}
if len(dirs) != 1 {
t.Fatal(dirs)
}
if dirs[0] != "." {
t.Fatal(dirs)
}
}
func TestWalkBiggerScript(t *testing.T) {
t.Skip("FIXME")
cc := &Compiler{}
script, err := cc.CompileScript("boot.cue", `
[
// {
// do: "load"
// from: {
// do: "local"
// dir: "ga"
// }
// },
{
do: "local"
dir: "bu"
},
{
do: "copy"
from: [
{
do: "local"
dir: "zo"
}
]
},
{
do: "exec"
args: ["ls"]
mount: "/mnt": input: [
{
do: "local"
dir: "meu"
}
]
}
]
`)
if err != nil {
t.Fatal(err)
}
dirs, err := script.LocalDirs()
if err != nil {
t.Fatal(err)
}
if len(dirs) != 4 {
t.Fatal(dirs)
}
wanted := "ga bu zo meu"
got := strings.Join(dirs, " ")
if wanted != got {
t.Fatal(got)
}
}

46
dagger/solver.go Normal file
View File

@ -0,0 +1,46 @@
package dagger
import (
"context"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
)
// Polyfill for buildkit gateway client
// Use instead of bkgw.Client
type Solver struct {
c bkgw.Client
}
func NewSolver(c bkgw.Client) Solver {
return Solver{
c: c,
}
}
func (s Solver) FS(input llb.State) FS {
return FS{
s: s,
input: input,
}
}
func (s Solver) Scratch() FS {
return s.FS(llb.Scratch())
}
func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
// marshal llb
def, err := st.Marshal(ctx, llb.LinuxAmd64)
if err != nil {
return nil, err
}
// call solve
res, err := s.c.Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()})
if err != nil {
return nil, err
}
// always use single reference (ignore multiple outputs & metadata)
return res.SingleRef()
}

30
dagger/spec.cue
View File

@ -41,6 +41,7 @@ package dagger
// The contents of a #dagger annotation // The contents of a #dagger annotation
#ComponentConfig: { #ComponentConfig: {
// FIXME: deprecated
input?: bool input?: bool
// script to compute the value // script to compute the value
@ -86,7 +87,7 @@ package dagger
#Script: [...#Op] #Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy
// Export a value from fs state to cue // Export a value from fs state to cue
#Export: { #Export: {
@ -96,22 +97,24 @@ package dagger
format: "json"|"yaml"|*"string"|"number"|"boolean" format: "json"|"yaml"|*"string"|"number"|"boolean"
} }
#Load: #LoadComponent| #LoadScript #Local: {
#LoadComponent: { do: "local"
do: "load" dir: string
from: #Component include: [...string] | *[]
}
#LoadScript: {
do: "load"
from: #Script
} }
// FIXME: bring back load (more efficient than copy)
#Load: {
do: "load"
from: #Component | #Script
}
#Exec: { #Exec: {
do: "exec" do: "exec"
args: [...string] args: [...string]
env: [string]: string env?: [string]: string
always: true | *false always?: true | *false
dir: string | *"/" dir: string | *"/"
mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
} }
@ -141,11 +144,10 @@ package dagger
#Copy: { #Copy: {
do: "copy" do: "copy"
from: #Script | #Component from: #Script | #Component
src: string | *"/" src?: string | *"/"
dest: string | *"/" dest?: string | *"/"
} }
#TestScript: #Script & [ #TestScript: #Script & [
{ do: "fetch-container", ref: "alpine:latest" }, { do: "fetch-container", ref: "alpine:latest" },
{ do: "exec", args: ["echo", "hello", "world" ] } { do: "exec", args: ["echo", "hello", "world" ] }

41
dagger/spec.go Normal file
View File

@ -0,0 +1,41 @@
package dagger
import (
"fmt"
"cuelang.org/go/cue"
cueerrors "cuelang.org/go/cue/errors"
)
// Cue spec validator
type Spec struct {
root *Value
}
// eg. Validate(op, "#Op")
func (s Spec) Validate(v *Value, defpath string) (err error) {
// Expand cue errors to get full details
// FIXME: there is probably a cleaner way to do this.
defer func() {
if err != nil {
err = fmt.Errorf("%s", cueerrors.Details(err, nil))
}
}()
def := s.root.LookupTarget(defpath)
if err := def.Err(); err != nil {
return err
}
if err := def.Unwrap().Fill(v).Validate(cue.Final()); err != nil {
return err
}
return nil
}
func (s Spec) Match(v *Value, defpath string) bool {
return s.Validate(v, defpath) == nil
}
func (s Spec) Get(target string) *Value {
return s.root.Get(target)
}

87
dagger/spec_test.go
View File

@ -2,8 +2,6 @@ package dagger
import ( import (
"testing" "testing"
"cuelang.org/go/cue"
) )
func TestMatch(t *testing.T) { func TestMatch(t *testing.T) {
@ -19,39 +17,40 @@ func TestMatch(t *testing.T) {
Src: `do: "fetch-git", remote: "github.com/shykes/tests"`, Src: `do: "fetch-git", remote: "github.com/shykes/tests"`,
Def: "#FetchGit", Def: "#FetchGit",
}, },
{ //FIXME: load is temporarily disabled
Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`, // {
Def: "#Load", // Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`,
}, // Def: "#Load",
{ // },
Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`, // {
Def: "#Load", // Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`,
}, // Def: "#Load",
// Make sure an empty op does NOT match // },
{ // // Make sure an empty op does NOT match
Src: ``, // {
Def: "", // Src: ``,
}, // Def: "",
{ // },
Src: `do: "load" // {
let package={bash:">3.0"} // Src: `do: "load"
from: foo //let package={bash:">3.0"}
let foo={#dagger: compute: [ //from: foo
{do: "fetch-container", ref: "alpine"}, //let foo={#dagger: compute: [
for pkg, info in package { // {do: "fetch-container", ref: "alpine"},
if (info & true) != _|_ { // for pkg, info in package {
do: "exec" // if (info & true) != _|_ {
args: ["echo", "hello", pkg] // do: "exec"
} // args: ["echo", "hello", pkg]
if (info & string) != _|_ { // }
do: "exec" // if (info & string) != _|_ {
args: ["echo", "hello", pkg, info] // do: "exec"
} // args: ["echo", "hello", pkg, info]
} // }
]} // }
`, //]}
Def: "#Load", //`,
}, // Def: "#Load",
// },
} }
for _, d := range data { for _, d := range data {
testMatch(t, d.Src, d.Def) testMatch(t, d.Src, d.Def)
@ -60,11 +59,11 @@ let foo={#dagger: compute: [
// Test an example op for false positives and negatives // Test an example op for false positives and negatives
func testMatch(t *testing.T, src interface{}, def string) { func testMatch(t *testing.T, src interface{}, def string) {
r := &Runtime{} cc := &Compiler{}
op := compile(t, r, src) op := compile(t, cc, src)
if def != "" { if def != "" {
if !r.matchSpec(op, def) { if err := op.Validate(def); err != nil {
t.Errorf("false negative: %s: %q", def, src) t.Errorf("false negative: %s: %q: %s", def, src, err)
} }
} }
for _, cmpDef := range []string{ for _, cmpDef := range []string{
@ -72,23 +71,23 @@ func testMatch(t *testing.T, src interface{}, def string) {
"#FetchGit", "#FetchGit",
"#FetchContainer", "#FetchContainer",
"#Export", "#Export",
"#Load",
"#Copy", "#Copy",
"#Local",
} { } {
if cmpDef == def { if cmpDef == def {
continue continue
} }
if r.matchSpec(op, cmpDef) { if err := op.Validate(cmpDef); err == nil {
t.Errorf("false positive: %s: %q", cmpDef, src) t.Errorf("false positive: %s: %q", cmpDef, src)
} }
} }
return return
} }
func compile(t *testing.T, r *Runtime, src interface{}) cue.Value { func compile(t *testing.T, cc *Compiler, src interface{}) *Value {
inst, err := r.Compile("", src) v, err := cc.Compile("", src)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
return inst.Value() return v
} }

53
dagger/state.go
View File

@ -1,53 +0,0 @@
package dagger
import (
"context"
"os"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
)
type State struct {
// Before last solve
input llb.State
// After last solve
output bkgw.Reference
// How to produce the output
s Solver
}
func NewState(s Solver) *State {
return &State{
input: llb.Scratch(),
s: s,
}
}
func (s *State) ReadFile(ctx context.Context, filename string) ([]byte, error) {
if s.output == nil {
return nil, os.ErrNotExist
}
return s.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename})
}
func (s *State) Change(ctx context.Context, op interface{}) error {
input := s.input
switch OP := op.(type) {
case llb.State:
input = OP
case func(llb.State) llb.State:
input = OP(input)
}
output, err := s.s.Solve(ctx, input)
if err != nil {
return err
}
s.input = input
s.output = output
return nil
}
func (s *State) LLB() llb.State {
return s.input
}

26
dagger/types.go Normal file
View File

@ -0,0 +1,26 @@
package dagger
import (
"context"
)
// Implemented by Component, Script, Op
type Executable interface {
Execute(context.Context, FS, Fillable) (FS, error)
Walk(func(*Op) error) error
}
// Something which can be filled in-place with a cue value
type Fillable interface {
Fill(interface{}) error
}
func Discard() Fillable {
return discard{}
}
type discard struct{}
func (d discard) Fill(x interface{}) error {
return nil
}

198
dagger/utils.go
View File

@ -4,44 +4,16 @@ import (
"crypto/rand" "crypto/rand"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"io/ioutil"
"os" "os"
"path/filepath"
"strings" "strings"
"cuelang.org/go/cue" "cuelang.org/go/cue"
cueAst "cuelang.org/go/cue/ast"
cueerrors "cuelang.org/go/cue/errors" cueerrors "cuelang.org/go/cue/errors"
cueformat "cuelang.org/go/cue/format" cueformat "cuelang.org/go/cue/format"
cueload "cuelang.org/go/cue/load"
cueParser "cuelang.org/go/cue/parser"
"github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/imagemetaresolver" "github.com/moby/buildkit/client/llb/imagemetaresolver"
"github.com/pkg/errors"
) )
// A nil equivalent for cue.Value (when returning errors)
var qnil cue.Value
type Fillable interface {
Fill(interface{}) error
}
func Discard() Fillable {
return discard{}
}
type discard struct{}
func (d discard) Fill(x interface{}) error {
return nil
}
type fillableValue struct {
root cue.Value
}
func cuePrint(v cue.Value) (string, error) { func cuePrint(v cue.Value) (string, error) {
b, err := cueformat.Node(v.Syntax()) b, err := cueformat.Node(v.Syntax())
if err != nil { if err != nil {
@ -50,68 +22,10 @@ func cuePrint(v cue.Value) (string, error) {
return string(b), nil return string(b), nil
} }
func (f *fillableValue) Fill(v interface{}) error {
root2 := f.root.Fill(v)
if err := root2.Err(); err != nil {
return err
}
f.root = root2
return nil
}
func cueScratch(r *cue.Runtime) Fillable {
f := &fillableValue{}
if inst, err := r.Compile("", ""); err == nil {
f.root = inst.Value()
}
return f
}
func cueErr(err error) error { func cueErr(err error) error {
return fmt.Errorf("%s", cueerrors.Details(err, &cueerrors.Config{})) return fmt.Errorf("%s", cueerrors.Details(err, &cueerrors.Config{}))
} }
func cueDecodeArray(a cue.Value, idx int, out interface{}) {
a.LookupPath(cue.MakePath(cue.Index(idx))).Decode(out)
}
func cueToJSON(v cue.Value) JSON {
var out JSON
v.Walk(
func(v cue.Value) bool {
b, err := v.MarshalJSON()
if err == nil {
newOut, err := out.Set(b, cuePathToStrings(v.Path())...)
if err == nil {
out = newOut
}
return false
}
return true
},
nil,
)
return out
}
// Build a cue instance from a directory and args
func cueBuild(r *cue.Runtime, cueRoot string, buildArgs ...string) (*cue.Instance, error) {
var err error
cueRoot, err = filepath.Abs(cueRoot)
if err != nil {
return nil, err
}
buildConfig := &cueload.Config{
ModuleRoot: cueRoot,
Dir: cueRoot,
}
instances := cueload.Instances(buildArgs, buildConfig)
if len(instances) != 1 {
return nil, errors.New("only one package is supported at a time")
}
return r.Build(instances[0])
}
func debugJSON(v interface{}) { func debugJSON(v interface{}) {
if os.Getenv("DEBUG") != "" { if os.Getenv("DEBUG") != "" {
e := json.NewEncoder(os.Stderr) e := json.NewEncoder(os.Stderr)
@ -144,104 +58,6 @@ func randomID(size int) (string, error) {
return fmt.Sprintf("%x", b), nil return fmt.Sprintf("%x", b), nil
} }
func cueWrapExpr(p string, v cueAst.Expr) (cueAst.Expr, error) {
pExpr, err := cueParser.ParseExpr("path", p)
if err != nil {
return v, err
}
out := v
cursor := pExpr
walk:
for {
switch c := cursor.(type) {
case *cueAst.SelectorExpr:
out = cueAst.NewStruct(
&cueAst.Field{
Value: out,
Label: c.Sel,
},
)
cursor = c.X
case *cueAst.Ident:
out = cueAst.NewStruct(
&cueAst.Field{
Value: out,
Label: c,
},
)
break walk
default:
return out, fmt.Errorf("invalid path expression: %q", p)
}
}
return out, nil
}
func cueWrapFile(p string, v interface{}) (*cueAst.File, error) {
f, err := cueParser.ParseFile("value", v)
if err != nil {
return f, err
}
decls := make([]cueAst.Decl, 0, len(f.Decls))
for _, decl := range f.Decls {
switch d := decl.(type) {
case *cueAst.Field:
wrappedExpr, err := cueWrapExpr(p, cueAst.NewStruct(d))
if err != nil {
return f, err
}
decls = append(decls, &cueAst.EmbedDecl{Expr: wrappedExpr})
case *cueAst.EmbedDecl:
wrappedExpr, err := cueWrapExpr(p, d.Expr)
if err != nil {
return f, err
}
d.Expr = wrappedExpr
decls = append(decls, d)
case *cueAst.ImportDecl:
decls = append(decls, decl)
default:
fmt.Printf("skipping unsupported decl type %#v\n\n", decl)
continue
}
}
f.Decls = decls
return f, nil
}
func cueIsEmptyStruct(v cue.Value) bool {
if st, err := v.Struct(); err == nil {
if st.Len() == 0 {
return true
}
}
return false
}
// Return false if v is not concrete, or contains any
// non-concrete fields or items.
func cueIsConcrete(v cue.Value) bool {
// FIXME: use Value.Walk?
if it, err := v.Fields(); err == nil {
for it.Next() {
if !cueIsConcrete(it.Value()) {
return false
}
}
return true
}
if it, err := v.List(); err == nil {
for it.Next() {
if !cueIsConcrete(it.Value()) {
return false
}
}
return true
}
dv, _ := v.Default()
return v.IsConcrete() || dv.IsConcrete()
}
// LLB Helper to pull a Docker image + all its metadata // LLB Helper to pull a Docker image + all its metadata
func llbDockerImage(ref string) llb.State { func llbDockerImage(ref string) llb.State {
return llb.Image( return llb.Image(
@ -272,17 +88,3 @@ func cueCleanPath(p string) (string, error) {
cp := cue.ParsePath(p) cp := cue.ParsePath(p)
return cp.String(), cp.Err() return cp.String(), cp.Err()
} }
func autoMarshal(value interface{}) ([]byte, error) {
switch v := value.(type) {
case []byte:
return v, nil
case string:
return []byte(v), nil
case io.Reader:
return ioutil.ReadAll(v)
default:
return nil, fmt.Errorf("unsupported marshal inoput type")
}
return []byte(fmt.Sprintf("%v", value)), nil
}

341
dagger/value.go Normal file
View File

@ -0,0 +1,341 @@
package dagger
import (
"fmt"
"os"
"cuelang.org/go/cue"
cueformat "cuelang.org/go/cue/format"
"github.com/moby/buildkit/client/llb"
)
// Polyfill for cue.Value.
// Use instead of cue.Value and cue.Instance
type Value struct {
// FIXME: don't embed, cleaner API
cue.Value
cc *Compiler
inst *cue.Instance
}
func (v *Value) Lock() {
if v.cc == nil {
return
}
v.cc.Lock()
}
func (v *Value) Unlock() {
if v.cc == nil {
return
}
v.cc.Unlock()
}
func (v *Value) Lookup(path ...string) *Value {
v.Lock()
defer v.Unlock()
return v.Wrap(v.Unwrap().Lookup(path...))
}
func (v *Value) LookupPath(p cue.Path) *Value {
v.Lock()
defer v.Unlock()
return v.Wrap(v.Unwrap().LookupPath(p))
}
// FIXME: deprecated by Get()
func (v *Value) LookupTarget(target string) *Value {
return v.LookupPath(cue.ParsePath(target))
}
func (v *Value) Get(target string) *Value {
return v.LookupPath(cue.ParsePath(target))
}
// Component returns the component value if v is a valid dagger component or an error otherwise.
// If no '#dagger' annotation is present, os.ErrNotExist
// is returned.
func (v *Value) Component() (*Component, error) {
c := &Component{
v: v,
}
if !c.Exists() {
return c, os.ErrNotExist
}
if err := c.Validate(); err != nil {
return c, err
}
return c, nil
}
func (v *Value) Script() (*Script, error) {
s := &Script{
v: v,
}
if err := s.Validate(); err != nil {
return s, err
}
return s, nil
}
func (v *Value) Executable() (Executable, error) {
if script, err := v.Script(); err == nil {
return script, nil
}
if component, err := v.Component(); err == nil {
return component, nil
}
if op, err := v.Op(); err == nil {
return op, nil
}
return nil, fmt.Errorf("value is not executable")
}
// ScriptOrComponent returns one of:
// (1) the component value if v is a valid component (type *Component)
// (2) the script value if v is a valid script (type *Script)
// (3) an error otherwise
func (v *Value) ScriptOrComponent() (interface{}, error) {
s, err := v.Script()
if err == nil {
return s, nil
}
c, err := v.Component()
if err == nil {
return c, nil
}
return nil, fmt.Errorf("not a script or component")
}
func (v *Value) Op() (*Op, error) {
// Merge #Op definition from spec to get default values
spec, err := v.Compiler().Spec()
if err != nil {
return nil, err
}
v, err = spec.Get("#Op").Merge(v)
if err != nil {
return nil, err
}
op := &Op{
v: v,
}
return op, nil
}
func (v *Value) Mount(dest string) (*Mount, error) {
mnt := &Mount{
v: v,
dest: dest,
}
return mnt, mnt.Validate()
}
// Interpret this value as a spec
func (v *Value) Spec() (*Spec, error) {
// Spec must be a struct
if _, err := v.Struct(); err != nil {
return nil, err
}
return &Spec{
root: v,
}, nil
}
// FIXME: receive string path?
func (v *Value) Merge(x interface{}, path ...string) (*Value, error) {
if xval, ok := x.(*Value); ok {
if xval.Compiler() != v.Compiler() {
return nil, fmt.Errorf("can't merge values from different compilers")
}
x = xval.Unwrap()
}
result := v.Wrap(v.Unwrap().Fill(x, path...))
return result, result.Validate()
}
func (v *Value) MergePath(x interface{}, p cue.Path) (*Value, error) {
// FIXME: array indexes and defs are not supported,
// they will be silently converted to regular fields.
// eg. `foo.#bar[0]` will become `foo["#bar"]["0"]`
return v.Merge(x, cuePathToStrings(p)...)
}
func (v *Value) MergeTarget(x interface{}, target string) (*Value, error) {
return v.MergePath(x, cue.ParsePath(target))
}
func (v *Value) RangeList(fn func(int, *Value) error) error {
it, err := v.List()
if err != nil {
return err
}
i := 0
for it.Next() {
if err := fn(i, v.Wrap(it.Value())); err != nil {
return err
}
i += 1
}
return nil
}
func (v *Value) RangeStruct(fn func(string, *Value) error) error {
it, err := v.Fields()
if err != nil {
return err
}
for it.Next() {
if err := fn(it.Label(), v.Wrap(it.Value())); err != nil {
return err
}
}
return nil
}
// Recursive concreteness check.
// Return false if v is not concrete, or contains any
// non-concrete fields or items.
func (v *Value) IsConcreteR() bool {
// FIXME: use Value.Walk
if it, err := v.Fields(); err == nil {
for it.Next() {
w := v.Wrap(it.Value())
if !w.IsConcreteR() {
return false
}
}
return true
}
if it, err := v.List(); err == nil {
for it.Next() {
w := v.Wrap(it.Value())
if !w.IsConcreteR() {
return false
}
}
return true
}
dv, _ := v.Default()
return v.IsConcrete() || dv.IsConcrete()
}
// Export concrete values to JSON. ignoring non-concrete values.
// Contrast with cue.Value.MarshalJSON which requires all values
// to be concrete.
func (v *Value) JSON() JSON {
v.Lock()
defer v.Unlock()
var out JSON
v.Walk(
func(v cue.Value) bool {
b, err := v.MarshalJSON()
if err == nil {
newOut, err := out.Set(b, cuePathToStrings(v.Path())...)
if err == nil {
out = newOut
}
return false
}
return true
},
nil,
)
return out
}
func (v *Value) SaveJSON(fs FS, filename string) FS {
return fs.Change(func(st llb.State) llb.State {
return st.File(
llb.Mkfile(filename, 0600, v.JSON()),
)
})
}
func (v *Value) Save(fs FS, filename string) (FS, error) {
src, err := v.Source()
if err != nil {
return fs, err
}
return fs.Change(func(st llb.State) llb.State {
return st.File(
llb.Mkfile(filename, 0600, src),
)
}), nil
}
func (v *Value) Validate(defs ...string) error {
if err := v.Unwrap().Validate(); err != nil {
return err
}
if len(defs) == 0 {
return nil
}
spec, err := v.Compiler().Spec()
if err != nil {
return err
}
for _, def := range defs {
if err := spec.Validate(v, def); err != nil {
return err
}
}
return nil
}
// Value implements Fillable.
// This is the only method which changes the value in-place.
// FIXME this co-exists awkwardly with the rest of Value.
func (v *Value) Fill(x interface{}) error {
v.Value = v.Value.Fill(x)
return v.Validate()
}
func (v *Value) Source() ([]byte, error) {
v.Lock()
defer v.Unlock()
return cueformat.Node(v.Eval().Syntax())
}
func (v *Value) IsEmptyStruct() bool {
if st, err := v.Struct(); err == nil {
if st.Len() == 0 {
return true
}
}
return false
}
func (v *Value) CueInst() *cue.Instance {
return v.inst
}
func (v *Value) Compiler() *Compiler {
if v.cc == nil {
return &Compiler{}
}
return v.cc
}
func (v *Value) Debugf(msg string, args ...interface{}) {
prefix := v.Path().String()
args = append([]interface{}{prefix}, args...)
debugf("%s: "+msg, args...)
}
func (v *Value) Wrap(v2 cue.Value) *Value {
return wrapValue(v2, v.inst, v.cc)
}
func (v *Value) Unwrap() cue.Value {
return v.Value
}
func wrapValue(v cue.Value, inst *cue.Instance, cc *Compiler) *Value {
return &Value{
Value: v,
cc: cc,
inst: inst,
}
}

35
dagger/value_test.go Normal file
View File

@ -0,0 +1,35 @@
package dagger
import (
"testing"
)
func TestSimple(t *testing.T) {
cc := &Compiler{}
_, err := cc.EmptyStruct()
if err != nil {
t.Fatal(err)
}
}
func TestCompileBootScript(t *testing.T) {
cc := &Compiler{}
s, err := cc.CompileScript("boot.cue", defaultBootScript)
if err != nil {
t.Fatal(err)
}
if err := s.Validate(); err != nil {
t.Fatal(err)
}
}
func TestCompileSimpleScript(t *testing.T) {
cc := &Compiler{}
s, err := cc.CompileScript("simple.cue", `[{do: "local", dir: "."}]`)
if err != nil {
t.Fatal(err)
}
if err := s.Validate(); err != nil {
t.Fatal(err)
}
}

43
main.go
View File

@ -1,43 +0,0 @@
// A simple main.go for testing the dagger Go API
package main
import (
"context"
"fmt"
"os"
"dagger.cloud/go/dagger"
)
func main() {
ctx := context.TODO()
c, err := dagger.NewClient(ctx, "")
if err != nil {
fatal(err)
}
configPath := "."
if len(os.Args) > 1 {
configPath = os.Args[1]
}
if err := c.SetConfig(configPath); err != nil {
fatal(err)
}
// if err := c.ConnectInput("source", os.Getenv("HOME")+"/Documents/github/samalba/hello-go"); err != nil {
// fatal(err)
// }
if err := c.Run(ctx, "compute"); err != nil {
fatal(err)
}
}
func fatalf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg, args...)
os.Exit(1)
}
func fatal(msg interface{}) {
fatalf("%s\n", msg)
}