Refactor

Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>

cmd/dagger/cmd/compute.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"os"
 
 	"dagger.cloud/go/dagger"
 	"dagger.cloud/go/dagger/ui"
@@ -16,32 +15,20 @@ var computeCmd = &cobra.Command{
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx := context.TODO()
-		c, err := dagger.NewClient(ctx, "")
+		// FIXME: boot and bootdir should be config fields, not args
+		c, err := dagger.NewClient(ctx, "", "", args[0])
 		if err != nil {
 			ui.Fatal(err)
 		}
-		target := args[0]
+		// FIXME: configure which config to compute (duh)
-		if target == "-" {
+		// FIXME: configure inputs
-			ui.Info("Assembling config from stdin\n")
-			// FIXME: include cue.mod/pkg from *somewhere* so stdin config can import
-			if err := c.SetConfig(os.Stdin); err != nil {
-				ui.Fatal(err)
-			}
-		} else {
-			ui.Info("Assembling config from %q\n", target)
-			if err := c.SetConfig(target); err != nil {
-				ui.Fatal(err)
-			}
-		}
 		ui.Info("Running")
-		output, err := c.Run(ctx, "compute")
+		output, err := c.Compute(ctx)
 		if err != nil {
 			ui.Fatal(err)
 		}
 		ui.Info("Processing output")
-		//		output.Print(os.Stdout)
 		fmt.Println(output.JSON())
-		// FIXME: write computed values back to env dir
 	},
 }
 

dagger/client.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"time"
@@ -15,14 +14,10 @@ import (
 
 	// Cue
 	"cuelang.org/go/cue"
-	cueerrors "cuelang.org/go/cue/errors"
-	cueformat "cuelang.org/go/cue/format"
 
 	// buildkit
 	bk "github.com/moby/buildkit/client"
 	_ "github.com/moby/buildkit/client/connhelper/dockercontainer"
-	"github.com/moby/buildkit/client/llb"
-	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 
 	// docker output
 	"github.com/containerd/console"
@@ -32,21 +27,34 @@ import (
 const (
 	defaultBuildkitHost = "docker-container://buildkitd"
 
-	bkConfigKey = "context"
+	bkBootKey  = "boot"
-	bkInputKey  = ":dagger:input:"
+	bkInputKey = "input"
-	bkActionKey = ":dagger:action:"
+
+	defaultBootDir = "."
+
+	// FIXME: rename to defaultConfig ?
+	defaultBootScript = `
+bootdir: string | *"."
+bootscript: [
+	{
+		do: "local"
+		dir: bootdir
+		include: ["*.cue", "cue.mod"]
+	},
+]
+`
 )
 
 type Client struct {
 	c *bk.Client
 
-	inputs    map[string]llb.State
 	localdirs map[string]string
-
+	boot      string
-	BKFrontend bkgw.BuildFunc
+	bootdir   string
+	input     string
 }
 
-func NewClient(ctx context.Context, host string) (*Client, error) {
+func NewClient(ctx context.Context, host, boot, bootdir string) (*Client, error) {
 	// buildkit client
 	if host == "" {
 		host = os.Getenv("BUILDKIT_HOST")
@@ -54,107 +62,59 @@ func NewClient(ctx context.Context, host string) (*Client, error) {
 	if host == "" {
 		host = defaultBuildkitHost
 	}
+	if boot == "" {
+		boot = defaultBootScript
+	}
+	if bootdir == "" {
+		bootdir = defaultBootDir
+	}
 	c, err := bk.New(ctx, host)
 	if err != nil {
 		return nil, errors.Wrap(err, "buildkit client")
 	}
 	return &Client{
 		c:         c,
-		inputs:    map[string]llb.State{},
+		boot:      boot,
+		bootdir:   bootdir,
+		input:     `{}`,
 		localdirs: map[string]string{},
 	}, nil
 }
 
-func (c *Client) ConnectInput(target string, input interface{}) error {
+func (c *Client) LocalDirs() ([]string, error) {
-	var st llb.State
+	boot, err := c.BootScript()
-	switch in := input.(type) {
+	if err != nil {
-	case llb.State:
+		return nil, err
-		st = in
-	case string:
-		// Generate a random local input label for security
-		st = c.AddLocalDir(in, target)
-	default:
-		return fmt.Errorf("unsupported input type")
 	}
-	c.inputs[bkInputKey+target] = st
+	return boot.LocalDirs()
-	return nil
 }
 
-func (c *Client) AddLocalDir(dir, label string, opts ...llb.LocalOption) llb.State {
+func (c *Client) BootScript() (*Script, error) {
-	c.localdirs[label] = dir
+	debugf("compiling boot script: %q\n", c.boot)
-	return llb.Local(label, opts...)
+	cc := &Compiler{}
-}
+	src, err := cc.Compile("boot.cue", c.boot)
-
+	if err != nil {
-// Set cue config for future calls.
+		return nil, errors.Wrap(err, "compile")
-// input can be:
-//   - llb.State: valid cue config directory
-//   - io.Reader: valid cue source
-//   - string: local path to valid cue file or directory
-//   - func(llb.State)llb.Stte: modify existing state
-
-func (c *Client) SetConfig(inputs ...interface{}) error {
-	for _, input := range inputs {
-		if err := c.setConfig(input); err != nil {
-			return err
-		}
 	}
-	return nil
+	src, err = src.MergeTarget(c.bootdir, "bootdir")
-}
+	if err != nil {
-
+		return nil, err
-func (c *Client) setConfig(input interface{}) error {
-	var st llb.State
-	switch in := input.(type) {
-	case llb.State:
-		st = in
-	case func(llb.State) llb.State:
-		// Modify previous state
-		last, ok := c.inputs[bkConfigKey]
-		if !ok {
-			last = llb.Scratch()
-		}
-		st = in(last)
-	case io.Reader:
-		contents, err := ioutil.ReadAll(in)
-		if err != nil {
-			return err
-		}
-		st = llb.Scratch().File(llb.Mkfile(
-			"config.cue",
-			0660,
-			contents,
-		))
-	// Interpret string as a path (dir or file)
-	case string:
-		info, err := os.Stat(in)
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			// FIXME: include pattern *.cue ooh yeah
-			st = c.AddLocalDir(in, "config",
-				//llb.IncludePatterns([]string{"*.cue", "cue.mod"})),
-				llb.FollowPaths([]string{"*.cue", "cue.mod"}),
-			)
-		} else {
-			f, err := os.Open(in)
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			return c.SetConfig(f)
-		}
 	}
-	c.inputs[bkConfigKey] = st
+	return src.Get("bootscript").Script()
-	return nil
 }
 
-func (c *Client) Run(ctx context.Context, action string) (*Output, error) {
+func (c *Client) Compute(ctx context.Context) (*Value, error) {
+	cc := &Compiler{}
+	out, err := cc.EmptyStruct()
+	if err != nil {
+		return nil, err
+	}
 	// Spawn Build() goroutine
 	eg, ctx := errgroup.WithContext(ctx)
 	events := make(chan *bk.SolveStatus)
 	outr, outw := io.Pipe()
 	// Spawn build function
-	eg.Go(c.buildfn(ctx, action, events, outw))
+	eg.Go(c.buildfn(ctx, events, outw))
 	// Spawn print function(s)
 	dispCtx := context.TODO()
 	var eventsdup chan *bk.SolveStatus
@@ -164,21 +124,36 @@ func (c *Client) Run(ctx context.Context, action string) (*Output, error) {
 	}
 	eg.Go(c.printfn(dispCtx, events, eventsdup))
 	// Retrieve output
-	out := NewOutput()
 	eg.Go(c.outputfn(ctx, outr, out))
 	return out, eg.Wait()
 }
 
-func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveStatus, w io.WriteCloser) func() error {
+func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.WriteCloser) func() error {
-	return func() error {
+	return func() (err error) {
-		defer debugf("buildfn complete")
+		defer func() {
+			debugf("buildfn complete, err=%q", err)
+			if err != nil {
+				// Close exporter pipe so that export processor can return
+				w.Close()
+			}
+		}()
+		boot, err := c.BootScript()
+		if err != nil {
+			close(ch)
+			return errors.Wrap(err, "assemble boot script")
+		}
+		bootSource, err := boot.Value().Source()
+		if err != nil {
+			close(ch)
+			return errors.Wrap(err, "serialize boot script")
+		}
 		// Setup solve options
 		opts := bk.SolveOpt{
 			FrontendAttrs: map[string]string{
-				bkActionKey: action,
+				bkInputKey: c.input,
+				bkBootKey:  string(bootSource),
 			},
-			LocalDirs:      c.localdirs,
+			LocalDirs: map[string]string{},
-			FrontendInputs: c.inputs,
 			// FIXME: catch output & return as cue value
 			Exports: []bk.ExportEntry{
 				{
@@ -189,16 +164,18 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt
 				},
 			},
 		}
-		// Setup frontend
+		// Connect local dirs
-		bkFrontend := c.BKFrontend
+		localdirs, err := c.LocalDirs()
-		if bkFrontend == nil {
+		if err != nil {
-			r := &Runtime{}
+			close(ch)
-			bkFrontend = r.BKFrontend
+			return errors.Wrap(err, "connect local dirs")
-		}
+		}
-		resp, err := c.c.Build(ctx, opts, "", bkFrontend, ch)
+		for _, dir := range localdirs {
+			opts.LocalDirs[dir] = dir
+		}
+		// Call buildkit solver
+		resp, err := c.c.Build(ctx, opts, "", Compute, ch)
 		if err != nil {
-			// Close exporter pipe so that export processor can return
-			w.Close()
 			err = errors.New(bkCleanError(err.Error()))
 			return errors.Wrap(err, "buildkit solve")
 		}
@@ -211,7 +188,7 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt
 }
 
 // Read tar export stream from buildkit Build(), and extract cue output
-func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func() error {
+func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Value) func() error {
 	return func() error {
 		defer debugf("outputfn complete")
 		tr := tar.NewReader(r)
@@ -229,14 +206,13 @@ func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func()
 				continue
 			}
 			debugf("outputfn: compiling & merging %q", h.Name)
-			// FIXME: only doing this for debug. you can pass tr directly as io.Reader.
+
-			contents, err := ioutil.ReadAll(tr)
+			cc := out.Compiler()
+			v, err := cc.Compile(h.Name, tr)
 			if err != nil {
 				return err
 			}
-			//if err := out.FillSource(h.Name, tr); err != nil {
+			if err := out.Fill(v); err != nil {
-			if err := out.FillSource(h.Name, contents); err != nil {
-				debugf("error with %s: contents=\n------\n%s\n-----\n", h.Name, contents)
 				return errors.Wrap(err, h.Name)
 			}
 			debugf("outputfn: DONE: compiling & merging %q", h.Name)
@@ -327,7 +303,7 @@ func (c *Client) printfn(ctx context.Context, ch, ch2 chan *bk.SolveStatus) func
 				for _, v := range status.Vertexes {
 					p := cue.ParsePath(v.Name)
 					if err := p.Err(); err != nil {
-						debugf("ignoring buildkit vertex %q: not a valid cue path", p.String())
+						debugf("ignoring buildkit vertex %q: not a valid cue path", v.Name)
 						continue
 					}
 					n := &Node{
@@ -377,58 +353,3 @@ func (c *Client) dockerprintfn(ctx context.Context, ch chan *bk.SolveStatus, out
 		return progressui.DisplaySolveStatus(ctx, "", cons, out, ch)
 	}
 }
-
-type Output struct {
-	r    *cue.Runtime
-	inst *cue.Instance
-}
-
-func NewOutput() *Output {
-	r := &cue.Runtime{}
-	inst, _ := r.Compile("", "")
-	return &Output{
-		r:    r,
-		inst: inst,
-	}
-}
-
-func (o *Output) Print(w io.Writer) error {
-	v := o.Cue().Value().Eval()
-	b, err := cueformat.Node(v.Syntax())
-	if err != nil {
-		return err
-	}
-	_, err = w.Write(b)
-	return err
-}
-
-func (o *Output) JSON() JSON {
-	return cueToJSON(o.Cue().Value())
-}
-
-func (o *Output) Cue() *cue.Instance {
-	return o.inst
-}
-
-func (o *Output) FillSource(filename string, x interface{}) error {
-	inst, err := o.r.Compile(filename, x)
-	if err != nil {
-		return fmt.Errorf("compile %s: %s", filename, cueerrors.Details(err, nil))
-	}
-	if err := o.FillValue(inst.Value()); err != nil {
-		return fmt.Errorf("merge %s: %s", filename, cueerrors.Details(err, nil))
-	}
-	return nil
-}
-
-func (o *Output) FillValue(x interface{}) error {
-	inst, err := o.inst.Fill(x)
-	if err != nil {
-		return err
-	}
-	if err := inst.Value().Validate(); err != nil {
-		return err
-	}
-	o.inst = inst
-	return nil
-}

dagger/compiler.go

@@ -0,0 +1,113 @@
+//go:generate sh gen.sh
+package dagger
+
+import (
+	"context"
+	"path"
+	"path/filepath"
+	"sync"
+
+	"cuelang.org/go/cue"
+	cueload "cuelang.org/go/cue/load"
+	"github.com/pkg/errors"
+)
+
+// Polyfill for a cue runtime
+// (we call it compiler to avoid confusion with dagger runtime)
+// Use this instead of cue.Runtime
+type Compiler struct {
+	sync.Mutex
+	cue.Runtime
+	spec *Spec
+}
+
+func (cc *Compiler) Cue() *cue.Runtime {
+	return &(cc.Runtime)
+}
+
+func (cc *Compiler) Spec() (*Spec, error) {
+	if cc.spec != nil {
+		return cc.spec, nil
+	}
+	v, err := cc.Compile("spec.cue", DaggerSpec)
+	if err != nil {
+		return nil, err
+	}
+	spec, err := v.Spec()
+	if err != nil {
+		return nil, err
+	}
+	cc.spec = spec
+	return spec, nil
+}
+
+// Compile an empty struct
+func (cc *Compiler) EmptyStruct() (*Value, error) {
+	return cc.Compile("", "")
+}
+
+func (cc *Compiler) Compile(name string, src interface{}) (*Value, error) {
+	inst, err := cc.Cue().Compile(name, src)
+	if err != nil {
+		// FIXME: cleaner way to unwrap cue error details?
+		return nil, cueErr(err)
+	}
+	return cc.Wrap(inst.Value(), inst), nil
+}
+
+func (cc *Compiler) CompileScript(name string, src interface{}) (*Script, error) {
+	v, err := cc.Compile(name, src)
+	if err != nil {
+		return nil, err
+	}
+	return v.Script()
+}
+
+// Build a cue configuration tree from the files in fs.
+func (cc *Compiler) Build(ctx context.Context, fs FS, args ...string) (*Value, error) {
+	debugf("Compiler.Build")
+	defer debugf("COMPLETE: Compiler.Build")
+	// The CUE overlay needs to be prefixed by a non-conflicting path with the
+	// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
+	// files it finds locally.
+	const overlayPrefix = "/config"
+
+	buildConfig := &cueload.Config{
+		Dir:     overlayPrefix,
+		Overlay: map[string]cueload.Source{},
+	}
+	buildArgs := args
+
+	err := fs.Walk(ctx, func(p string, f Stat) error {
+		debugf("  Compiler.Build: processing %q", p)
+		if f.IsDir() {
+			return nil
+		}
+		if filepath.Ext(p) != ".cue" {
+			return nil
+		}
+		contents, err := fs.ReadFile(ctx, p)
+		if err != nil {
+			return errors.Wrap(err, p)
+		}
+		overlayPath := path.Join(overlayPrefix, p)
+		buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	instances := cueload.Instances(buildArgs, buildConfig)
+	if len(instances) != 1 {
+		return nil, errors.New("only one package is supported at a time")
+	}
+	inst, err := cc.Cue().Build(instances[0])
+	if err != nil {
+		return nil, err
+	}
+	return cc.Wrap(inst.Value(), inst), nil
+}
+
+func (cc *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value {
+	return wrapValue(v, inst, cc)
+}

dagger/component.go

@@ -0,0 +1,66 @@
+package dagger
+
+import (
+	"context"
+)
+
+type Component struct {
+	v *Value
+}
+
+func (c *Component) Value() *Value {
+	return c.v
+}
+
+func (c *Component) Exists() bool {
+	// Does #dagger exist?
+	if c.Config().Err() != nil {
+		return false
+	}
+	return true
+}
+
+// Return the contents of the "#dagger" annotation.
+func (c *Component) Config() *Value {
+	return c.Value().Get("#dagger")
+}
+
+// Verify that this component respects the dagger component spec.
+//
+// NOTE: calling matchSpec("#Component") is not enough because
+//   it does not match embedded scalars.
+func (c Component) Validate() error {
+	return c.Config().Validate("#ComponentConfig")
+}
+
+// Return this component's compute script.
+func (c Component) ComputeScript() (*Script, error) {
+	return c.Value().Get("#dagger.compute").Script()
+}
+
+// Compute the configuration for this component.
+// Note that we simply execute the underlying compute script from an
+// empty filesystem state.
+// (It is never correct to pass an input filesystem state to compute a component)
+func (c *Component) Compute(ctx context.Context, s Solver, out Fillable) (FS, error) {
+	return c.Execute(ctx, s.Scratch(), out)
+}
+
+// A component implements the Executable interface by returning its
+// compute script.
+// See Value.Executable().
+func (c *Component) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	script, err := c.ComputeScript()
+	if err != nil {
+		return fs, err
+	}
+	return script.Execute(ctx, fs, out)
+}
+
+func (c *Component) Walk(fn func(*Op) error) error {
+	script, err := c.ComputeScript()
+	if err != nil {
+		return err
+	}
+	return script.Walk(fn)
+}

dagger/compute.go

@@ -0,0 +1,41 @@
+package dagger
+
+import (
+	"context"
+	"fmt"
+
+	cueerrors "cuelang.org/go/cue/errors"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+)
+
+// Buildkit compute entrypoint (BK calls if "solve" or "build")
+// Use by wrapping in a buildkit client Build call, or buildkit frontend.
+func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
+	// FIXME: wrap errors to avoid crashing buildkit Build()
+	//    with cue error types (why??)
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s", cueerrors.Details(err, nil))
+			debugf("execute returned an error. Wrapping...")
+		}
+	}()
+	debugf("initializing env")
+	env, err := NewEnv(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+	debugf("computing env")
+	// Compute output overlay
+	if err := env.Compute(ctx); err != nil {
+		return nil, err
+	}
+	debugf("exporting env")
+	// Export env to a cue directory
+	outdir := NewSolver(c).Scratch()
+	outdir, err = env.Export(outdir)
+	if err != nil {
+		return nil, err
+	}
+	// Wrap cue directory in buildkit result
+	return outdir.Result(ctx)
+}

dagger/env.go

@@ -0,0 +1,186 @@
+package dagger
+
+import (
+	"context"
+	"os"
+
+	"cuelang.org/go/cue"
+	cueflow "cuelang.org/go/tools/flow"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/pkg/errors"
+)
+
+type Env struct {
+	// Base config
+	base *Value
+	// Input overlay: user settings, external directories, secrets...
+	input *Value
+
+	// Output overlay: computed values, generated directories
+	output *Value
+
+	// Buildkit solver
+	s Solver
+
+	// shared cue compiler
+	// (because cue API requires shared runtime for everything)
+	cc *Compiler
+}
+
+// Initialize a new environment
+func NewEnv(ctx context.Context, c bkgw.Client) (*Env, error) {
+	cc := &Compiler{}
+	// 1. Load base config (specified by client)
+	debugf("Loading base configuration")
+	base, err := envBase(ctx, c, cc)
+	if err != nil {
+		return nil, err
+	}
+	// 2. Load input overlay (specified by client)
+	debugf("Loading input overlay")
+	input, err := envInput(ctx, c, cc)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := base.Merge(input); err != nil {
+		return nil, errors.Wrap(err, "merge base & input")
+	}
+	return &Env{
+		base:  base,
+		input: input,
+		s:     NewSolver(c),
+		cc:    cc,
+	}, nil
+}
+
+// Compute missing values in env configuration, and write them to state.
+func (env *Env) Compute(ctx context.Context) error {
+	debugf("Computing environment")
+	output, err := env.Walk(ctx, func(c *Component, out Fillable) error {
+		_, err := c.Compute(ctx, env.s, out)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	env.output = output
+	return nil
+}
+
+// Export env to a directory of cue files
+// (Use with FS.Change)
+func (env *Env) Export(fs FS) (FS, error) {
+	// FIXME: we serialize as JSON to guarantee a self-contained file.
+	//   Value.Save() leaks imports, so requires a shared cue.mod with
+	//   client which is undesirable.
+	//   Once Value.Save() resolves non-builtin imports with a tree shake,
+	//   we can use it here.
+	fs = env.base.SaveJSON(fs, "base.cue")
+	fs = env.input.SaveJSON(fs, "input.cue")
+	if env.output != nil {
+		fs = env.output.SaveJSON(fs, "output.cue")
+	}
+	return fs, nil
+}
+
+type EnvWalkFunc func(*Component, Fillable) error
+
+// Walk components and return any computed values
+func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
+	debugf("Env.Walk")
+	defer debugf("COMPLETE: Env.Walk")
+	// Cueflow cue instance
+	// FIXME: make this cleaner in *Value by keeping intermediary instances
+	flowInst, err := env.base.CueInst().Fill(env.input.CueInst())
+	if err != nil {
+		return nil, err
+	}
+	// Initialize empty output
+	out, err := env.cc.EmptyStruct()
+	if err != nil {
+		return nil, err
+	}
+	// Cueflow config
+	flowCfg := &cueflow.Config{
+		UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error {
+			debugf("compute step")
+			if t == nil {
+				return nil
+			}
+			debugf("cueflow task %q: %s", t.Path().String(), t.State().String())
+			if t.State() != cueflow.Terminated {
+				return nil
+			}
+			debugf("cueflow task %q: filling result", t.Path().String())
+			// Merge task value into output
+			var err error
+			// FIXME: does cueflow.Task.Value() contain only filled values,
+			// or base + filled?
+			out, err = out.MergePath(t.Value(), t.Path())
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+	// Cueflow match func
+	flowMatchFn := func(v cue.Value) (cueflow.Runner, error) {
+		val := env.cc.Wrap(v, flowInst)
+		c, err := val.Component()
+		if os.IsNotExist(err) {
+			// Not a component: skip
+			return nil, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		return cueflow.RunnerFunc(func(t *cueflow.Task) error {
+			return fn(c, t)
+		}), nil
+	}
+	// Orchestrate execution with cueflow
+	flow := cueflow.New(flowCfg, flowInst, flowMatchFn)
+	if err := flow.Run(ctx); err != nil {
+		return out, err
+	}
+	return out, nil
+}
+
+func envBase(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
+	// 1. Receive boot script from client.
+	debugf("retrieving boot script")
+	bootSrc, exists := c.BuildOpts().Opts["boot"]
+	if !exists {
+		// No boot script: return empty base config
+		return cc.EmptyStruct()
+	}
+
+	// 2. Compile & execute boot script
+	debugf("compiling boot script")
+	boot, err := cc.CompileScript("boot.cue", bootSrc)
+	if err != nil {
+		return nil, errors.Wrap(err, "compile boot script")
+	}
+	debugf("executing boot script")
+	bootState, err := boot.Execute(ctx, NewSolver(c).Scratch(), Discard())
+	if err != nil {
+		return nil, errors.Wrap(err, "execute boot script")
+	}
+	// 3. load cue files produced by bootstrap script
+	// FIXME: BuildAll() to force all files (no required package..)
+	debugf("building cue configuration from boot state")
+	base, err := cc.Build(ctx, bootState)
+	debugf("done building cue configuration: err=%q", err)
+	return base, err
+}
+
+func envInput(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
+	// 1. Receive input overlay from client.
+	//     This is used to provide run-time settings, directories..
+	inputSrc, exists := c.BuildOpts().Opts["input"]
+	if !exists {
+		// No input overlay: return empty tree
+		return cc.EmptyStruct()
+	}
+	return cc.Compile("input.cue", inputSrc)
+}

dagger/fs.go

@@ -0,0 +1,143 @@
+package dagger
+
+import (
+	"context"
+	"path"
+
+	"github.com/moby/buildkit/client/llb"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	fstypes "github.com/tonistiigi/fsutil/types"
+)
+
+type Stat struct {
+	*fstypes.Stat
+}
+
+type FS struct {
+	// Before last solve
+	input llb.State
+	// After last solve
+	output bkgw.Reference
+	// How to produce the output
+	s Solver
+}
+
+func (fs FS) Solver() Solver {
+	return fs.s
+}
+
+// Compute output from input, if not done already.
+//   This method uses a pointer receiver to simplify
+//   calling it, since it is called in almost every
+//   other method.
+func (fs *FS) solve(ctx context.Context) error {
+	if fs.output != nil {
+		return nil
+	}
+	output, err := fs.s.Solve(ctx, fs.input)
+	if err != nil {
+		return err
+	}
+	fs.output = output
+	return nil
+}
+
+func (fs FS) ReadFile(ctx context.Context, filename string) ([]byte, error) {
+	// Lazy solve
+	if err := (&fs).solve(ctx); err != nil {
+		return nil, err
+	}
+	return fs.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename})
+}
+
+func (fs FS) ReadDir(ctx context.Context, dir string) ([]Stat, error) {
+	// Lazy solve
+	if err := (&fs).solve(ctx); err != nil {
+		return nil, err
+	}
+	st, err := fs.output.ReadDir(ctx, bkgw.ReadDirRequest{
+		Path: dir,
+	})
+	if err != nil {
+		return nil, err
+	}
+	out := make([]Stat, len(st))
+	for i := range st {
+		out[i] = Stat{
+			Stat: st[i],
+		}
+	}
+	return out, nil
+}
+
+func (fs FS) walk(ctx context.Context, p string, fn WalkFunc) error {
+	files, err := fs.ReadDir(ctx, p)
+	if err != nil {
+		return err
+	}
+	for _, f := range files {
+		fPath := path.Join(p, f.GetPath())
+		if err := fn(fPath, f); err != nil {
+			return err
+		}
+		if f.IsDir() {
+			if err := fs.walk(ctx, fPath, fn); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+type WalkFunc func(string, Stat) error
+
+func (fs FS) Walk(ctx context.Context, fn WalkFunc) error {
+	// Lazy solve
+	if err := (&fs).solve(ctx); err != nil {
+		return err
+	}
+	return fs.walk(ctx, "/", fn)
+}
+
+type ChangeFunc func(llb.State) llb.State
+
+func (fs FS) Change(changes ...ChangeFunc) FS {
+	for _, change := range changes {
+		fs = fs.Set(change(fs.input))
+	}
+	return fs
+}
+
+func (fs FS) Set(st llb.State) FS {
+	fs.input = st
+	fs.output = nil
+	return fs
+}
+
+func (fs FS) Solve(ctx context.Context) (FS, error) {
+	if err := (&fs).solve(ctx); err != nil {
+		return fs, err
+	}
+	return fs, nil
+}
+
+func (fs FS) LLB() llb.State {
+	return fs.input
+}
+
+func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) {
+	if err := (&fs).solve(ctx); err != nil {
+		return nil, err
+	}
+	return fs.output, nil
+}
+
+func (fs FS) Result(ctx context.Context) (*bkgw.Result, error) {
+	res := bkgw.NewResult()
+	ref, err := fs.Ref(ctx)
+	if err != nil {
+		return nil, err
+	}
+	res.SetRef(ref)
+	return res, nil
+}

dagger/gen.go

@@ -46,6 +46,7 @@ package dagger
 
 // The contents of a #dagger annotation
 #ComponentConfig: {
+	// FIXME: deprecated
 	input?: bool
 
 	// script to compute the value
@@ -91,7 +92,7 @@ package dagger
 #Script: [...#Op]
 
 // One operation in a script
-#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy
+#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy
 
 // Export a value from fs state to cue
 #Export: {
@@ -101,22 +102,24 @@ package dagger
 	format: "json"|"yaml"|*"string"|"number"|"boolean"
 }
 
-#Load: #LoadComponent| #LoadScript
+#Local: {
-#LoadComponent: {
+	do: "local"
-	do: "load"
+	dir: string
-	from: #Component
+	include: [...string] | *[]
-}
-#LoadScript: {
-	do: "load"
-	from: #Script
 }
 
+// FIXME: bring back load (more efficient than copy)
+
+#Load: {
+	do: "load"
+	from: #Component | #Script
+}
 
 #Exec: {
 	do: "exec"
 	args: [...string]
-	env: [string]: string
+	env?: [string]: string
-	always: true | *false
+	always?: true | *false
 	dir: string | *"/"
 	mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
 }
@@ -146,11 +149,10 @@ package dagger
 #Copy: {
 	do: "copy"
 	from: #Script | #Component
-	src: string | *"/"
+	src?: string | *"/"
-	dest: string | *"/"
+	dest?: string | *"/"
 }
 
-
 #TestScript: #Script & [
 	{ do: "fetch-container", ref: "alpine:latest" },
 	{ do: "exec", args: ["echo", "hello", "world" ] }

dagger/job.go

@@ -1,326 +0,0 @@
-package dagger
-
-import (
-	"context"
-	"fmt"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"cuelang.org/go/cue"
-	cueerrors "cuelang.org/go/cue/errors"
-	cueload "cuelang.org/go/cue/load"
-	cueflow "cuelang.org/go/tools/flow"
-	"github.com/moby/buildkit/client/llb"
-	bkgw "github.com/moby/buildkit/frontend/gateway/client"
-	"github.com/pkg/errors"
-)
-
-type Solver interface {
-	Solve(context.Context, llb.State) (bkgw.Reference, error)
-}
-
-// 1 buildkit build = 1 job
-type Job struct {
-	c bkgw.Client
-	// needed for cue operations
-	r *Runtime
-}
-
-// Execute and wrap the result in a buildkit result
-func (job Job) BKExecute(ctx context.Context) (_r *bkgw.Result, _e error) {
-	debugf("Executing bk frontend")
-	// wrap errors to avoid crashing buildkit with cue error types (why??)
-	defer func() {
-		if _e != nil {
-			_e = fmt.Errorf("%s", cueerrors.Details(_e, nil))
-			debugf("execute returned an error. Wrapping...")
-		}
-	}()
-	out, err := job.Execute(ctx)
-	if err != nil {
-		return nil, err
-	}
-	// encode job output to buildkit result
-	debugf("[runtime] job executed. Encoding output")
-	// FIXME: we can't serialize result to standalone cue (with no imports).
-	// So the client cannot safely compile output without access to the same cue.mod
-	// as the runtime (which we don't want).
-	// So for now we return the output as json, still parsed as cue on the client
-	// to keep our options open. Once there is a "tree shake" primitive, we can
-	// use that to return cue.
-	//
-	// Uncomment to return actual cue:
-	// ----
-	// outbytes, err := cueformat.Node(out.Value().Eval().Syntax())
-	// if err != nil {
-	// 	return nil, err
-	// }
-	// ----
-	outbytes := cueToJSON(out.Value())
-	debugf("[runtime] output encoded. Writing output to exporter")
-	outref, err := job.Solve(ctx,
-		llb.Scratch().File(llb.Mkfile("computed.cue", 0600, outbytes)),
-	)
-	if err != nil {
-		return nil, err
-	}
-	debugf("[runtime] output written to exporter. returning to buildkit solver")
-	res := bkgw.NewResult()
-	res.SetRef(outref)
-	return res, nil
-}
-
-func (job Job) Execute(ctx context.Context) (_i *cue.Instance, _e error) {
-	debugf("[runtime] Execute()")
-	defer func() { debugf("[runtime] DONE Execute(): err=%v", _e) }()
-	state, err := job.Config(ctx)
-	if err != nil {
-		return nil, err
-	}
-	// Merge input information into the cue config
-	inputs, err := job.Inputs(ctx)
-	if err != nil {
-		return nil, err
-	}
-	for target := range inputs {
-		// FIXME: cleaner code generation, missing cue.Value.FillPath
-		state, err = job.r.fill(state, `#dagger: input: true`, target)
-		if err != nil {
-			return nil, errors.Wrapf(err, "connect input %q", target)
-		}
-	}
-	action := job.Action()
-	switch action {
-	case "compute":
-		return job.doCompute(ctx, state)
-	case "export":
-		return job.doExport(ctx, state)
-	default:
-		return job.doExport(ctx, state)
-	}
-}
-
-func (job Job) doExport(ctx context.Context, state *cue.Instance) (*cue.Instance, error) {
-	return state, nil
-}
-
-func (job Job) doCompute(ctx context.Context, state *cue.Instance) (*cue.Instance, error) {
-	out, err := job.r.Compile("computed.cue", "")
-	if err != nil {
-		return nil, err
-	}
-	// Setup cueflow
-	debugf("Setting up cueflow")
-	flow := cueflow.New(
-		&cueflow.Config{
-			UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error {
-				debugf("cueflow event")
-				if t == nil {
-					return nil
-				}
-				debugf("cueflow task %q: %s", t.Path().String(), t.State().String())
-				if t.State() == cueflow.Terminated {
-					debugf("cueflow task %q: filling result", t.Path().String())
-					out, err = out.Fill(t.Value(), cuePathToStrings(t.Path())...)
-					if err != nil {
-						return err
-					}
-					// FIXME: catch merge errors early with state
-				}
-				return nil
-			},
-		},
-		state,
-		// Task match func
-		func(v cue.Value) (cueflow.Runner, error) {
-			// Is v a component (has #dagger) with a field 'compute' ?
-			isComponent, err := job.r.isComponent(v, "compute")
-			if err != nil {
-				return nil, err
-			}
-			if !isComponent {
-				return nil, nil
-			}
-			debugf("[%s] component detected\n", v.Path().String())
-			// task runner func
-			runner := cueflow.RunnerFunc(func(t *cueflow.Task) error {
-				computeScript := t.Value().LookupPath(cue.ParsePath("#dagger.compute"))
-				script, err := job.newScript(computeScript)
-				if err != nil {
-					return err
-				}
-				// Run the script & fill the result into the task
-				return script.Run(ctx, t)
-			})
-			return runner, nil
-		},
-	)
-	debugf("Running cueflow")
-	if err := flow.Run(ctx); err != nil {
-		return nil, err
-	}
-	debugf("Completed cueflow run. Merging result.")
-	state, err = state.Fill(out)
-	if err != nil {
-		return nil, err
-	}
-	debugf("Result merged")
-	// Return only the computed values
-	return out, nil
-}
-
-func (job Job) bk() bkgw.Client {
-	return job.c
-}
-
-func (job Job) Action() string {
-	opts := job.bk().BuildOpts().Opts
-	if action, ok := opts[bkActionKey]; ok {
-		return action
-	}
-	return ""
-}
-
-// Load the cue config for this job
-// (received as llb input)
-func (job Job) Config(ctx context.Context) (*cue.Instance, error) {
-	src := llb.Local(bkConfigKey,
-		llb.SessionID(job.bk().BuildOpts().SessionID),
-		llb.SharedKeyHint(bkConfigKey),
-		llb.WithCustomName("load config"),
-	)
-
-	bkInputs, err := job.bk().Inputs(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if st, ok := bkInputs[bkConfigKey]; ok {
-		src = st
-	}
-	// job.runDebug(ctx, src, "ls", "-la", "/mnt")
-	return job.LoadCue(ctx, src)
-}
-
-func (job Job) runDebug(ctx context.Context, mnt llb.State, args ...string) error {
-	opts := []llb.RunOption{
-		llb.Args(args),
-		llb.AddMount("/mnt", mnt),
-	}
-	cmd := llb.Image("alpine").Run(opts...).Root()
-	ref, err := job.Solve(ctx, cmd)
-	if err != nil {
-		return errors.Wrap(err, "debug")
-	}
-	// force non-lazy solve
-	if _, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{Path: "/"}); err != nil {
-		return errors.Wrap(err, "debug")
-	}
-	return nil
-}
-
-func (job Job) Inputs(ctx context.Context) (map[string]llb.State, error) {
-	bkInputs, err := job.bk().Inputs(ctx)
-	if err != nil {
-		return nil, err
-	}
-	inputs := map[string]llb.State{}
-	for key, input := range bkInputs {
-		if !strings.HasPrefix(key, bkInputKey) {
-			continue
-		}
-		target := strings.Replace(key, bkInputKey, "", 1)
-		targetPath := cue.ParsePath(target)
-		if err := targetPath.Err(); err != nil {
-			return nil, errors.Wrapf(err, "input target %q", target)
-		}
-		// FIXME: check that the path can be passed to Fill
-		//  (eg. only regular fields, no array indexes, no defs)
-		// see cuePathToStrings
-		inputs[target] = input
-	}
-	return inputs, nil
-}
-
-// loadFiles recursively loads all .cue files from a buildkit gateway
-// FIXME: this is highly inefficient.
-func loadFiles(ctx context.Context, ref bkgw.Reference, p, overlayPrefix string, overlay map[string]cueload.Source) error {
-	// FIXME: we cannot use `IncludePattern` here, otherwise sub directories
-	// (e.g. "cue.mod") will be skipped.
-	files, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{
-		Path: p,
-	})
-	if err != nil {
-		return err
-	}
-	for _, f := range files {
-		fPath := path.Join(p, f.GetPath())
-		if f.IsDir() {
-			if err := loadFiles(ctx, ref, fPath, overlayPrefix, overlay); err != nil {
-				return err
-			}
-			continue
-		}
-
-		if filepath.Ext(fPath) != ".cue" {
-			continue
-		}
-
-		contents, err := ref.ReadFile(ctx, bkgw.ReadRequest{
-			Filename: fPath,
-		})
-		if err != nil {
-			return errors.Wrap(err, f.GetPath())
-		}
-		overlay[path.Join(overlayPrefix, fPath)] = cueload.FromBytes(contents)
-	}
-
-	return nil
-}
-
-func (job Job) LoadCue(ctx context.Context, st llb.State, args ...string) (*cue.Instance, error) {
-	// The CUE overlay needs to be prefixed by a non-conflicting path with the
-	// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
-	// files it finds locally.
-	const overlayPrefix = "/config"
-
-	buildConfig := &cueload.Config{
-		Dir:     overlayPrefix,
-		Overlay: map[string]cueload.Source{},
-	}
-	buildArgs := args
-
-	// Inject cue files from llb state into overlay
-	ref, err := job.Solve(ctx, st)
-	if err != nil {
-		return nil, err
-	}
-	if err := loadFiles(ctx, ref, ".", overlayPrefix, buildConfig.Overlay); err != nil {
-		return nil, err
-	}
-
-	instances := cueload.Instances(buildArgs, buildConfig)
-	if len(instances) != 1 {
-		return nil, errors.New("only one package is supported at a time")
-	}
-	inst, err := job.r.Build(instances[0])
-	if err != nil {
-		return nil, cueErr(err)
-	}
-	return inst, nil
-}
-
-func (job Job) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
-	// marshal llb
-	def, err := st.Marshal(ctx, llb.LinuxAmd64)
-	if err != nil {
-		return nil, err
-	}
-	// call solve
-	res, err := job.bk().Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()})
-	if err != nil {
-		return nil, err
-	}
-	// always use single reference (ignore multiple outputs & metadata)
-	return res.SingleRef()
-}

dagger/mount.go

@@ -0,0 +1,38 @@
+package dagger
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/pkg/errors"
+)
+
+type Mount struct {
+	dest string
+	v    *Value
+}
+
+func (mnt *Mount) Validate(defs ...string) error {
+	return mnt.v.Validate(append(defs, "#Mount")...)
+}
+
+func (mnt *Mount) LLB(ctx context.Context, s Solver) (llb.RunOption, error) {
+	if err := mnt.Validate("#MountTmp"); err == nil {
+		return llb.AddMount(mnt.dest, llb.Scratch(), llb.Tmpfs()), nil
+	}
+	if err := mnt.Validate("#MountCache"); err == nil {
+		// FIXME: cache mount
+		return nil, fmt.Errorf("FIXME: cache mount not yet implemented")
+	}
+	// Compute source component or script, discarding fs writes & output value
+	from, err := mnt.v.Lookup("from").Executable()
+	if err != nil {
+		return nil, errors.Wrap(err, "from")
+	}
+	fromFS, err := from.Execute(ctx, s.Scratch(), Discard())
+	if err != nil {
+		return nil, err
+	}
+	return llb.AddMount(mnt.dest, fromFS.LLB()), nil
+}

dagger/op.go

@@ -0,0 +1,256 @@
+package dagger
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/pkg/errors"
+)
+
+type Op struct {
+	v *Value
+}
+
+func (op *Op) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	action, err := op.Action()
+	if err != nil {
+		return fs, err
+	}
+	return action(ctx, fs, out)
+}
+
+func (op *Op) Walk(fn func(*Op) error) error {
+	debugf("Walk %v", op.v)
+	isCopy := (op.Validate("#Copy") == nil)
+	isLoad := (op.Validate("#Load") == nil)
+	if isCopy || isLoad {
+		debugf("MATCH %v", op.v)
+		if from, err := op.Get("from").Executable(); err == nil {
+			if err := from.Walk(fn); err != nil {
+				return err
+			}
+		}
+		// FIXME: we tolerate "from" which is not executable
+	}
+	if err := op.Validate("#Exec"); err == nil {
+		return op.Get("mount").RangeStruct(func(k string, v *Value) error {
+			if from, err := op.Get("from").Executable(); err == nil {
+				if err := from.Walk(fn); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+	// depth first
+	return fn(op)
+}
+
+type Action func(context.Context, FS, Fillable) (FS, error)
+
+func (op *Op) Action() (Action, error) {
+	// An empty struct is allowed as a no-op, to be
+	//  more tolerant of if() in arrays.
+	if op.v.IsEmptyStruct() {
+		return op.Nothing, nil
+	}
+	// Manually check for a 'do' field.
+	// This is necessary because Runtime.ValidateSpec has a bug
+	// where an empty struct matches everything.
+	// see https://github.com/cuelang/cue/issues/566#issuecomment-749735878
+	// Once the bug is fixed, the manual check can be removed.
+	if _, err := op.Get("do").String(); err != nil {
+		return nil, fmt.Errorf("invalid action: no \"do\" field")
+	}
+	actions := map[string]Action{
+		"#Copy":           op.Copy,
+		"#Exec":           op.Exec,
+		"#Export":         op.Export,
+		"#FetchContainer": op.FetchContainer,
+		"#FetchGit":       op.FetchGit,
+		"#Local":          op.Local,
+	}
+	for def, action := range actions {
+		if err := op.Validate(def); err == nil {
+			op.v.Debugf("OP MATCH: %s", def)
+			return action, nil
+		}
+	}
+	return nil, fmt.Errorf("invalid operation: %s", op.v.JSON())
+}
+
+func (op *Op) Validate(defs ...string) error {
+	defs = append(defs, "#Op")
+	return op.v.Validate(defs...)
+}
+
+func (op *Op) Copy(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	// Decode copy options
+	src, err := op.Get("src").String()
+	if err != nil {
+		return fs, err
+	}
+	dest, err := op.Get("dest").String()
+	if err != nil {
+		return fs, err
+	}
+	from, err := op.Get("from").Executable()
+	if err != nil {
+		return fs, errors.Wrap(err, "from")
+	}
+	// Compute source component or script, discarding fs writes & output value
+	fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard())
+	if err != nil {
+		return fs, err
+	}
+	return fs.Change(func(st llb.State) llb.State {
+		return st.File(llb.Copy(
+			fromFS.LLB(),
+			src,
+			dest,
+			// FIXME: allow more configurable llb options
+			// For now we define the following convenience presets:
+			&llb.CopyInfo{
+				CopyDirContentsOnly: true,
+				CreateDestPath:      true,
+				AllowWildcard:       true,
+			},
+		))
+	}), nil // lazy solve
+}
+
+func (op *Op) Nothing(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	return fs, nil
+}
+func (op *Op) Local(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	dir, err := op.Get("dir").String()
+	if err != nil {
+		return fs, err
+	}
+	var include []string
+	if err := op.Get("include").Decode(&include); err != nil {
+		return fs, err
+	}
+	return fs.Set(llb.Local(dir, llb.FollowPaths(include))), nil // lazy solve
+}
+
+func (op *Op) Exec(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	var opts []llb.RunOption
+	var cmd struct {
+		Args   []string
+		Env    map[string]string
+		Dir    string
+		Always bool
+	}
+	op.v.Decode(&cmd)
+	// marker for status events
+	// FIXME
+	opts = append(opts, llb.WithCustomName(op.v.Path().String()))
+	// args
+	opts = append(opts, llb.Args(cmd.Args))
+	// dir
+	dir := cmd.Dir
+	if dir == "" {
+		dir = "/"
+	}
+	// env
+	for k, v := range cmd.Env {
+		opts = append(opts, llb.AddEnv(k, v))
+	}
+	// always?
+	if cmd.Always {
+		cacheBuster, err := randomID(8)
+		if err != nil {
+			return fs, err
+		}
+		opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster))
+	}
+	// mounts
+	if err := op.v.RangeStruct(func(k string, v *Value) error {
+		mnt, err := v.Mount(k)
+		if err != nil {
+			return err
+		}
+		opt, err := mnt.LLB(ctx, fs.Solver())
+		if err != nil {
+			return err
+		}
+		opts = append(opts, opt)
+		return nil
+	}); err != nil {
+		return fs, err
+	}
+	// --> Execute
+	return fs.Change(func(st llb.State) llb.State {
+		return st.Run(opts...).Root()
+	}), nil // lazy solve
+}
+
+func (op *Op) Export(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	source, err := op.Get("source").String()
+	if err != nil {
+		return fs, err
+	}
+	format, err := op.Get("format").String()
+	if err != nil {
+		return fs, err
+	}
+	contents, err := fs.ReadFile(ctx, source)
+	if err != nil {
+		return fs, errors.Wrapf(err, "export %s", source)
+	}
+	switch format {
+	case "string":
+		if err := out.Fill(string(contents)); err != nil {
+			return fs, err
+		}
+	case "json":
+		var o interface{}
+		if err := json.Unmarshal(contents, &o); err != nil {
+			return fs, err
+		}
+		if err := out.Fill(o); err != nil {
+			return fs, err
+		}
+	default:
+		return fs, fmt.Errorf("unsupported export format: %q", format)
+	}
+	return fs, nil
+}
+
+func (op *Op) Load(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	from, err := op.Get("from").Executable()
+	if err != nil {
+		return fs, errors.Wrap(err, "load")
+	}
+	fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard())
+	if err != nil {
+		return fs, errors.Wrap(err, "load: compute source")
+	}
+	return fs.Set(fromFS.LLB()), nil
+}
+
+func (op *Op) FetchContainer(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	ref, err := op.Get("ref").String()
+	if err != nil {
+		return fs, err
+	}
+	return fs.Set(llb.Image(ref)), nil
+}
+func (op *Op) FetchGit(ctx context.Context, fs FS, out Fillable) (FS, error) {
+	remote, err := op.Get("remote").String()
+	if err != nil {
+		return fs, err
+	}
+	ref, err := op.Get("ref").String()
+	if err != nil {
+		return fs, err
+	}
+	return fs.Set(llb.Git(remote, ref)), nil // lazy solve
+}
+
+func (op *Op) Get(target string) *Value {
+	return op.v.Get(target)
+}

dagger/op_test.go

@@ -0,0 +1,32 @@
+package dagger
+
+import (
+	"testing"
+)
+
+func TestCopyMatch(t *testing.T) {
+	cc := &Compiler{}
+	src := `do: "copy", from: [{do: "local", dir: "foo"}]`
+	v, err := cc.Compile("", src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	op, err := v.Op()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := op.Validate("#Copy"); err != nil {
+		t.Fatal(err)
+	}
+	n := 0
+	err = op.Walk(func(op *Op) error {
+		n += 1
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatal(n)
+	}
+}

dagger/runtime.go

@@ -1,109 +0,0 @@
-//go:generate sh gen.sh
-package dagger
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	"cuelang.org/go/cue"
-	cueerrors "cuelang.org/go/cue/errors"
-	bkgw "github.com/moby/buildkit/frontend/gateway/client"
-	"github.com/pkg/errors"
-)
-
-type Runtime struct {
-	l sync.Mutex
-
-	cue.Runtime
-}
-
-func (r *Runtime) Cue() *cue.Runtime {
-	return &(r.Runtime)
-}
-
-func (r *Runtime) fill(inst *cue.Instance, v interface{}, target string) (*cue.Instance, error) {
-	targetPath := cue.ParsePath(target)
-	if err := targetPath.Err(); err != nil {
-		return nil, err
-	}
-	p := cuePathToStrings(targetPath)
-	if src, ok := v.(string); ok {
-		vinst, err := r.Compile(target, src)
-		if err != nil {
-			return nil, err
-		}
-		return inst.Fill(vinst.Value(), p...)
-	}
-	return inst.Fill(v, p...)
-}
-
-// func (r Runtime) Run(...)
-// Buildkit run entrypoint
-func (r *Runtime) BKFrontend(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
-	return r.newJob(c).BKExecute(ctx)
-}
-
-func (r *Runtime) newJob(c bkgw.Client) Job {
-	return Job{
-		r: r,
-		c: c,
-	}
-}
-
-// Check whether a value is a valid component
-// FIXME: calling matchSpec("#Component") is not enough because
-//   it does not match embedded scalars.
-func (r *Runtime) isComponent(v cue.Value, fields ...string) (bool, error) {
-	cfg := v.LookupPath(cue.ParsePath("#dagger"))
-	if cfg.Err() != nil {
-		// No "#dagger" -> not a component
-		return false, nil
-	}
-	for _, field := range fields {
-		if cfg.Lookup(field).Err() != nil {
-			return false, nil
-		}
-	}
-	if err := r.validateSpec(cfg, "#ComponentConfig"); err != nil {
-		return true, errors.Wrap(err, "invalid #dagger")
-	}
-	return true, nil
-}
-
-// eg. validateSpec(op, "#Op")
-// eg. validateSpec(dag, "#DAG")
-func (r *Runtime) validateSpec(v cue.Value, defpath string) (err error) {
-	// Expand cue errors to get full details
-	// FIXME: there is probably a cleaner way to do this.
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf("%s", cueerrors.Details(err, nil))
-		}
-	}()
-	r.l.Lock()
-	defer r.l.Unlock()
-
-	// FIXME cache spec instance
-	spec, err := r.Compile("dagger.cue", DaggerSpec)
-	if err != nil {
-		panic("invalid spec")
-	}
-	def := spec.Value().LookupPath(cue.ParsePath(defpath))
-	if err := def.Err(); err != nil {
-		return err
-	}
-	v = v.Eval()
-	if err := v.Validate(); err != nil {
-		return err
-	}
-	res := def.Unify(v)
-	if err := res.Validate(cue.Final()); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (r *Runtime) matchSpec(v cue.Value, def string) bool {
-	return r.validateSpec(v, def) == nil
-}

dagger/script.go

@@ -2,325 +2,69 @@ package dagger
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 
-	"cuelang.org/go/cue"
-	"github.com/moby/buildkit/client/llb"
 	"github.com/pkg/errors"
 )
 
 type Script struct {
-	v   cue.Value
+	v *Value
-	job Job
-
-	// current state
-	state *State
 }
 
-func (job Job) newScript(v cue.Value) (*Script, error) {
+func (s Script) Validate() error {
-	s := &Script{
+	return s.Value().Validate("#Script")
-		v:     v,
-		job:   job,
-		state: NewState(job),
-	}
-	if err := s.Validate(); err != nil {
-		return nil, s.err(err, "invalid script")
-	}
-	return s, nil
 }
 
-type Action func(context.Context, cue.Value, Fillable) error
+func (s *Script) Value() *Value {
-
-func (s *Script) Run(ctx context.Context, out Fillable) error {
-	op, err := s.Cue().List()
-	if err != nil {
-		return s.err(err, "run")
-	}
-	i := 0
-	for op.Next() {
-		// If op is not concrete, interrupt execution without error.
-		// This allows gradual resolution: compute what you can compute.. leave the rest incomplete.
-		if !cueIsConcrete(op.Value()) {
-			debugf("%s: non-concrete op. Leaving script unfinished", op.Value().Path().String())
-			return nil
-		}
-		if err := s.Do(ctx, op.Value(), out); err != nil {
-			return s.err(err, "run op %d", i+1)
-		}
-		i += 1
-	}
-	return nil
-}
-
-func (s *Script) Do(ctx context.Context, op cue.Value, out Fillable) error {
-	// Skip no-ops without error (allows more flexible use of if())
-	// FIXME: maybe not needed once a clear pattern is established for
-	// how to use if() in a script?
-	if cueIsEmptyStruct(op) {
-		return nil
-	}
-	actions := map[string]Action{
-		// "#Copy": s.copy,
-		"#Exec":           s.exec,
-		"#Export":         s.export,
-		"#FetchContainer": s.fetchContainer,
-		"#FetchGit":       s.fetchGit,
-		"#Load":           s.load,
-		"#Copy":           s.copy,
-	}
-	for def, action := range actions {
-		if s.matchSpec(op, def) {
-			debugf("OP MATCH: %s: %s: %v", def, op.Path().String(), op)
-			return action(ctx, op, out)
-		}
-	}
-	return fmt.Errorf("[%s] invalid operation: %s", s.v.Path().String(), cueToJSON(op))
-}
-
-func (s *Script) copy(ctx context.Context, v cue.Value, out Fillable) error {
-	// Decode copy options
-	var op struct {
-		Src  string
-		Dest string
-	}
-	if err := v.Decode(&op); err != nil {
-		return err
-	}
-	from := v.Lookup("from")
-	if isComponent, err := s.job.r.isComponent(from); err != nil {
-		return err
-	} else if isComponent {
-		return s.copyComponent(ctx, from, op.Src, op.Dest)
-	}
-	if s.matchSpec(from, "#Script") {
-		return s.copyScript(ctx, from, op.Src, op.Dest)
-	}
-	return fmt.Errorf("copy: invalid source")
-}
-
-func (s *Script) copyScript(ctx context.Context, from cue.Value, src, dest string) error {
-	// Load source script
-	fromScript, err := s.job.newScript(from)
-	if err != nil {
-		return err
-	}
-	// Execute source script
-	if err := fromScript.Run(ctx, Discard()); err != nil {
-		return err
-	}
-	return s.State().Change(ctx, func(st llb.State) llb.State {
-		return st.File(llb.Copy(
-			fromScript.State().LLB(),
-			src,
-			dest,
-			// FIXME: allow more configurable llb options
-			// For now we define the following convenience presets:
-			&llb.CopyInfo{
-				CopyDirContentsOnly: true,
-				CreateDestPath:      true,
-				AllowWildcard:       true,
-			},
-		))
-	})
-}
-
-func (s *Script) copyComponent(ctx context.Context, from cue.Value, src, dest string) error {
-	return s.copyScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute")), src, dest)
-}
-
-func (s *Script) load(ctx context.Context, op cue.Value, out Fillable) error {
-	from := op.Lookup("from")
-	isComponent, err := s.job.r.isComponent(from)
-	if err != nil {
-		return err
-	}
-	if isComponent {
-		debugf("LOAD: from is a component")
-		return s.loadScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute")))
-	}
-	if s.matchSpec(from, "#Script") {
-		return s.loadScript(ctx, from)
-	}
-	return fmt.Errorf("load: invalid source")
-}
-
-func (s *Script) loadScript(ctx context.Context, v cue.Value) error {
-	from, err := s.job.newScript(v)
-	if err != nil {
-		return errors.Wrap(err, "load")
-	}
-	// NOTE we discard cue outputs from running the loaded script.
-	// This means we load the LLB state but NOT the cue exports.
-	// In other words: cue exports are always private to their original location.
-	if err := from.Run(ctx, Discard()); err != nil {
-		return errors.Wrap(err, "load/execute")
-	}
-	// overwrite buildkit state from loaded from
-	s.state = from.state
-	return nil
-}
-
-func (s *Script) exec(ctx context.Context, v cue.Value, out Fillable) error {
-	var opts []llb.RunOption
-	var cmd struct {
-		Args   []string
-		Env    map[string]string
-		Dir    string
-		Always bool
-	}
-	v.Decode(&cmd)
-	// marker for status events
-	opts = append(opts, llb.WithCustomName(v.Path().String()))
-	// args
-	opts = append(opts, llb.Args(cmd.Args))
-	// dir
-	dir := cmd.Dir
-	if dir == "" {
-		dir = "/"
-	}
-	// env
-	for k, v := range cmd.Env {
-		opts = append(opts, llb.AddEnv(k, v))
-	}
-	// always?
-	if cmd.Always {
-		cacheBuster, err := randomID(8)
-		if err != nil {
-			return err
-		}
-		opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster))
-	}
-	// mounts
-	mnt, _ := v.Lookup("mount").Fields()
-	for mnt.Next() {
-		opt, err := s.mount(ctx, mnt.Label(), mnt.Value())
-		if err != nil {
-			return err
-		}
-		opts = append(opts, opt)
-	}
-	// --> Execute
-	return s.State().Change(ctx, func(st llb.State) llb.State {
-		return st.Run(opts...).Root()
-	})
-}
-
-func (s *Script) mount(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
-	if s.matchSpec(source, "#MountTmp") {
-		return llb.AddMount(dest, llb.Scratch(), llb.Tmpfs()), nil
-	}
-	if s.matchSpec(source, "#MountCache") {
-		// FIXME: cache mount
-		return nil, fmt.Errorf("FIXME: cache mount not yet implemented")
-	}
-	if s.matchSpec(source, "#MountScript") {
-		return s.mountScript(ctx, dest, source)
-	}
-	if s.matchSpec(source, "#MountComponent") {
-		return s.mountComponent(ctx, dest, source)
-	}
-	return nil, fmt.Errorf("mount %s to %s: invalid source", source.Path().String(), dest)
-}
-
-// mount when the input is a script (see mountComponent, mountTmpfs, mountCache)
-func (s *Script) mountScript(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
-	script, err := s.job.newScript(source)
-	if err != nil {
-		return nil, err
-	}
-	// FIXME: this is where we re-run everything,
-	// and rely on solver cache / dedup
-	if err := script.Run(ctx, Discard()); err != nil {
-		return nil, err
-	}
-	return llb.AddMount(dest, script.State().LLB()), nil
-}
-
-func (s *Script) mountComponent(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) {
-	return s.mountScript(ctx, dest, source.LookupPath(cue.ParsePath("from.#dagger.compute")))
-}
-
-func (s *Script) fetchContainer(ctx context.Context, v cue.Value, out Fillable) error {
-	var op struct {
-		Ref string
-	}
-	if err := v.Decode(&op); err != nil {
-		return errors.Wrap(err, "decode fetch-container")
-	}
-	return s.State().Change(ctx, llb.Image(op.Ref))
-}
-
-func (s *Script) fetchGit(ctx context.Context, v cue.Value, out Fillable) error {
-	// See #FetchGit in spec.cue
-	var op struct {
-		Remote string
-		Ref    string
-	}
-	if err := v.Decode(&op); err != nil {
-		return errors.Wrap(err, "decode fetch-git")
-	}
-	return s.State().Change(ctx, llb.Git(op.Remote, op.Ref))
-}
-
-func (s *Script) export(ctx context.Context, v cue.Value, out Fillable) error {
-	// See #Export in spec.cue
-	var op struct {
-		Source string
-		// FIXME: target
-		// Target string
-		Format string
-	}
-	v.Decode(&op)
-	b, err := s.State().ReadFile(ctx, op.Source)
-	if err != nil {
-		return err
-	}
-	switch op.Format {
-	case "string":
-		return out.Fill(string(b))
-	case "json":
-		var o interface{}
-		if err := json.Unmarshal(b, &o); err != nil {
-			return err
-		}
-		return out.Fill(o)
-	default:
-		return fmt.Errorf("unsupported export format: %q", op.Format)
-	}
-}
-
-func (s *Script) Cue() cue.Value {
 	return s.v
 }
 
-func (s *Script) Location() string {
+// Run a dagger script
-	return s.Cue().Path().String()
+func (s *Script) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) {
-}
+	err := s.v.RangeList(func(idx int, v *Value) error {
-
+		// If op not concrete, interrupt without error.
-func (s *Script) err(e error, msg string, args ...interface{}) error {
+		// This allows gradual resolution:
-	return errors.Wrapf(e, s.Location()+": "+msg, args...)
+		//    compute what you can compute.. leave the rest incomplete.
-}
+		if !v.IsConcreteR() {
-
+			return nil
-func (s *Script) Validate() error {
-	return s.job.r.validateSpec(s.Cue(), "#Script")
-}
-
-func (s *Script) State() *State {
-	return s.state
-}
-
-func (s *Script) matchSpec(v cue.Value, def string) bool {
-	// FIXME: we manually filter out empty structs to avoid false positives
-	// This is necessary because Runtime.ValidateSpec has a bug
-	// where an empty struct matches everything.
-	// see https://github.com/cuelang/cue/issues/566#issuecomment-749735878
-	// Once the bug is fixed, the manual check can be removed.
-	if st, err := v.Struct(); err == nil {
-		if st.Len() == 0 {
-			debugf("FIXME: manually filtering out empty struct from spec match")
-			return false
 		}
-	}
+		op, err := v.Op()
-	return s.job.r.matchSpec(v, def)
+		if err != nil {
+			return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len())
+		}
+		fs, err = op.Execute(ctx, fs, out)
+		if err != nil {
+			return errors.Wrapf(err, "execute op %d/%d", idx+1, s.v.Len())
+		}
+		return nil
+	})
+	return fs, err
+}
+
+func (s *Script) Walk(fn func(op *Op) error) error {
+	return s.v.RangeList(func(idx int, v *Value) error {
+		op, err := v.Op()
+		if err != nil {
+			return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len())
+		}
+		if err := op.Walk(fn); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+func (s *Script) LocalDirs() ([]string, error) {
+	var dirs []string
+	err := s.Walk(func(op *Op) error {
+		if err := op.Validate("#Local"); err != nil {
+			return nil
+		}
+		dir, err := op.Get("dir").String()
+		if err != nil {
+			return err
+		}
+		dirs = append(dirs, dir)
+		return nil
+	})
+	return dirs, err
 }

dagger/script_test.go

@@ -0,0 +1,78 @@
+package dagger
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestWalkBootScript(t *testing.T) {
+	cc := &Compiler{}
+	script, err := cc.CompileScript("boot.cue", defaultBootScript)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dirs, err := script.LocalDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(dirs) != 1 {
+		t.Fatal(dirs)
+	}
+	if dirs[0] != "." {
+		t.Fatal(dirs)
+	}
+}
+
+func TestWalkBiggerScript(t *testing.T) {
+	t.Skip("FIXME")
+	cc := &Compiler{}
+	script, err := cc.CompileScript("boot.cue", `
+[
+//	{
+//		do: "load"
+//		from: {
+//			do: "local"
+//			dir: "ga"
+//		}
+//	},
+	{
+		do: "local"
+		dir: "bu"
+	},
+	{
+		do: "copy"
+		from: [
+			{
+				do: "local"
+				dir: "zo"
+			}
+		]
+	},
+	{
+		do: "exec"
+		args: ["ls"]
+		mount: "/mnt": input: [
+			{
+				do: "local"
+				dir: "meu"
+			}
+		]
+	}
+]
+`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dirs, err := script.LocalDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(dirs) != 4 {
+		t.Fatal(dirs)
+	}
+	wanted := "ga bu zo meu"
+	got := strings.Join(dirs, " ")
+	if wanted != got {
+		t.Fatal(got)
+	}
+}

dagger/solver.go

@@ -0,0 +1,46 @@
+package dagger
+
+import (
+	"context"
+
+	"github.com/moby/buildkit/client/llb"
+	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+)
+
+// Polyfill for buildkit gateway client
+// Use instead of bkgw.Client
+type Solver struct {
+	c bkgw.Client
+}
+
+func NewSolver(c bkgw.Client) Solver {
+	return Solver{
+		c: c,
+	}
+}
+
+func (s Solver) FS(input llb.State) FS {
+	return FS{
+		s:     s,
+		input: input,
+	}
+}
+
+func (s Solver) Scratch() FS {
+	return s.FS(llb.Scratch())
+}
+
+func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
+	// marshal llb
+	def, err := st.Marshal(ctx, llb.LinuxAmd64)
+	if err != nil {
+		return nil, err
+	}
+	// call solve
+	res, err := s.c.Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()})
+	if err != nil {
+		return nil, err
+	}
+	// always use single reference (ignore multiple outputs & metadata)
+	return res.SingleRef()
+}

dagger/spec.cue

@@ -41,6 +41,7 @@ package dagger
 
 // The contents of a #dagger annotation
 #ComponentConfig: {
+	// FIXME: deprecated
 	input?: bool
 
 	// script to compute the value
@@ -86,7 +87,7 @@ package dagger
 #Script: [...#Op]
 
 // One operation in a script
-#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy
+#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy
 
 // Export a value from fs state to cue
 #Export: {
@@ -96,22 +97,24 @@ package dagger
 	format: "json"|"yaml"|*"string"|"number"|"boolean"
 }
 
-#Load: #LoadComponent| #LoadScript
+#Local: {
-#LoadComponent: {
+	do: "local"
-	do: "load"
+	dir: string
-	from: #Component
+	include: [...string] | *[]
-}
-#LoadScript: {
-	do: "load"
-	from: #Script
 }
 
+// FIXME: bring back load (more efficient than copy)
+
+#Load: {
+	do: "load"
+	from: #Component | #Script
+}
 
 #Exec: {
 	do: "exec"
 	args: [...string]
-	env: [string]: string
+	env?: [string]: string
-	always: true | *false
+	always?: true | *false
 	dir: string | *"/"
 	mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript
 }
@@ -141,11 +144,10 @@ package dagger
 #Copy: {
 	do: "copy"
 	from: #Script | #Component
-	src: string | *"/"
+	src?: string | *"/"
-	dest: string | *"/"
+	dest?: string | *"/"
 }
 
-
 #TestScript: #Script & [
 	{ do: "fetch-container", ref: "alpine:latest" },
 	{ do: "exec", args: ["echo", "hello", "world" ] }

dagger/spec.go

@@ -0,0 +1,41 @@
+package dagger
+
+import (
+	"fmt"
+
+	"cuelang.org/go/cue"
+	cueerrors "cuelang.org/go/cue/errors"
+)
+
+// Cue spec validator
+type Spec struct {
+	root *Value
+}
+
+// eg. Validate(op, "#Op")
+func (s Spec) Validate(v *Value, defpath string) (err error) {
+	// Expand cue errors to get full details
+	// FIXME: there is probably a cleaner way to do this.
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s", cueerrors.Details(err, nil))
+		}
+	}()
+
+	def := s.root.LookupTarget(defpath)
+	if err := def.Err(); err != nil {
+		return err
+	}
+	if err := def.Unwrap().Fill(v).Validate(cue.Final()); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s Spec) Match(v *Value, defpath string) bool {
+	return s.Validate(v, defpath) == nil
+}
+
+func (s Spec) Get(target string) *Value {
+	return s.root.Get(target)
+}

dagger/spec_test.go

@@ -2,8 +2,6 @@ package dagger
 
 import (
 	"testing"
-
-	"cuelang.org/go/cue"
 )
 
 func TestMatch(t *testing.T) {
@@ -19,39 +17,40 @@ func TestMatch(t *testing.T) {
 			Src: `do: "fetch-git", remote: "github.com/shykes/tests"`,
 			Def: "#FetchGit",
 		},
-		{
+		//FIXME: load is temporarily disabled
-			Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`,
+		//		{
-			Def: "#Load",
+		//			Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`,
-		},
+		//			Def: "#Load",
-		{
+		//		},
-			Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`,
+		//		{
-			Def: "#Load",
+		//			Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`,
-		},
+		//			Def: "#Load",
-		// Make sure an empty op does NOT match
+		//		},
-		{
+		//		// Make sure an empty op does NOT match
-			Src: ``,
+		//		{
-			Def: "",
+		//			Src: ``,
-		},
+		//			Def: "",
-		{
+		//		},
-			Src: `do: "load"
+		//		{
-let package={bash:">3.0"}
+		//			Src: `do: "load"
-from: foo
+		//let package={bash:">3.0"}
-let foo={#dagger: compute: [
+		//from: foo
-	{do: "fetch-container", ref: "alpine"},
+		//let foo={#dagger: compute: [
-	for pkg, info in package {
+		//	{do: "fetch-container", ref: "alpine"},
-		if (info & true) != _|_ {
+		//	for pkg, info in package {
-			do: "exec"
+		//		if (info & true) != _|_ {
-			args: ["echo", "hello", pkg]
+		//			do: "exec"
-		}
+		//			args: ["echo", "hello", pkg]
-		if (info & string) != _|_ {
+		//		}
-			do: "exec"
+		//		if (info & string) != _|_ {
-			args: ["echo", "hello", pkg, info]
+		//			do: "exec"
-		}
+		//			args: ["echo", "hello", pkg, info]
-	}
+		//		}
-]}
+		//	}
-`,
+		//]}
-			Def: "#Load",
+		//`,
-		},
+		//			Def: "#Load",
+		//		},
 	}
 	for _, d := range data {
 		testMatch(t, d.Src, d.Def)
@@ -60,11 +59,11 @@ let foo={#dagger: compute: [
 
 // Test an example op for false positives and negatives
 func testMatch(t *testing.T, src interface{}, def string) {
-	r := &Runtime{}
+	cc := &Compiler{}
-	op := compile(t, r, src)
+	op := compile(t, cc, src)
 	if def != "" {
-		if !r.matchSpec(op, def) {
+		if err := op.Validate(def); err != nil {
-			t.Errorf("false negative: %s: %q", def, src)
+			t.Errorf("false negative: %s: %q: %s", def, src, err)
 		}
 	}
 	for _, cmpDef := range []string{
@@ -72,23 +71,23 @@ func testMatch(t *testing.T, src interface{}, def string) {
 		"#FetchGit",
 		"#FetchContainer",
 		"#Export",
-		"#Load",
 		"#Copy",
+		"#Local",
 	} {
 		if cmpDef == def {
 			continue
 		}
-		if r.matchSpec(op, cmpDef) {
+		if err := op.Validate(cmpDef); err == nil {
 			t.Errorf("false positive: %s: %q", cmpDef, src)
 		}
 	}
 	return
 }
 
-func compile(t *testing.T, r *Runtime, src interface{}) cue.Value {
+func compile(t *testing.T, cc *Compiler, src interface{}) *Value {
-	inst, err := r.Compile("", src)
+	v, err := cc.Compile("", src)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return inst.Value()
+	return v
 }

dagger/state.go

@@ -1,53 +0,0 @@
-package dagger
-
-import (
-	"context"
-	"os"
-
-	"github.com/moby/buildkit/client/llb"
-	bkgw "github.com/moby/buildkit/frontend/gateway/client"
-)
-
-type State struct {
-	// Before last solve
-	input llb.State
-	// After last solve
-	output bkgw.Reference
-	// How to produce the output
-	s Solver
-}
-
-func NewState(s Solver) *State {
-	return &State{
-		input: llb.Scratch(),
-		s:     s,
-	}
-}
-
-func (s *State) ReadFile(ctx context.Context, filename string) ([]byte, error) {
-	if s.output == nil {
-		return nil, os.ErrNotExist
-	}
-	return s.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename})
-}
-
-func (s *State) Change(ctx context.Context, op interface{}) error {
-	input := s.input
-	switch OP := op.(type) {
-	case llb.State:
-		input = OP
-	case func(llb.State) llb.State:
-		input = OP(input)
-	}
-	output, err := s.s.Solve(ctx, input)
-	if err != nil {
-		return err
-	}
-	s.input = input
-	s.output = output
-	return nil
-}
-
-func (s *State) LLB() llb.State {
-	return s.input
-}

dagger/types.go

@@ -0,0 +1,26 @@
+package dagger
+
+import (
+	"context"
+)
+
+// Implemented by Component, Script, Op
+type Executable interface {
+	Execute(context.Context, FS, Fillable) (FS, error)
+	Walk(func(*Op) error) error
+}
+
+// Something which can be filled in-place with a cue value
+type Fillable interface {
+	Fill(interface{}) error
+}
+
+func Discard() Fillable {
+	return discard{}
+}
+
+type discard struct{}
+
+func (d discard) Fill(x interface{}) error {
+	return nil
+}

dagger/utils.go

@@ -4,44 +4,16 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"cuelang.org/go/cue"
-	cueAst "cuelang.org/go/cue/ast"
 	cueerrors "cuelang.org/go/cue/errors"
 	cueformat "cuelang.org/go/cue/format"
-	cueload "cuelang.org/go/cue/load"
-	cueParser "cuelang.org/go/cue/parser"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/client/llb/imagemetaresolver"
-	"github.com/pkg/errors"
 )
 
-// A nil equivalent for cue.Value (when returning errors)
-var qnil cue.Value
-
-type Fillable interface {
-	Fill(interface{}) error
-}
-
-func Discard() Fillable {
-	return discard{}
-}
-
-type discard struct{}
-
-func (d discard) Fill(x interface{}) error {
-	return nil
-}
-
-type fillableValue struct {
-	root cue.Value
-}
-
 func cuePrint(v cue.Value) (string, error) {
 	b, err := cueformat.Node(v.Syntax())
 	if err != nil {
@@ -50,68 +22,10 @@ func cuePrint(v cue.Value) (string, error) {
 	return string(b), nil
 }
 
-func (f *fillableValue) Fill(v interface{}) error {
-	root2 := f.root.Fill(v)
-	if err := root2.Err(); err != nil {
-		return err
-	}
-	f.root = root2
-	return nil
-}
-
-func cueScratch(r *cue.Runtime) Fillable {
-	f := &fillableValue{}
-	if inst, err := r.Compile("", ""); err == nil {
-		f.root = inst.Value()
-	}
-	return f
-}
-
 func cueErr(err error) error {
 	return fmt.Errorf("%s", cueerrors.Details(err, &cueerrors.Config{}))
 }
 
-func cueDecodeArray(a cue.Value, idx int, out interface{}) {
-	a.LookupPath(cue.MakePath(cue.Index(idx))).Decode(out)
-}
-
-func cueToJSON(v cue.Value) JSON {
-	var out JSON
-	v.Walk(
-		func(v cue.Value) bool {
-			b, err := v.MarshalJSON()
-			if err == nil {
-				newOut, err := out.Set(b, cuePathToStrings(v.Path())...)
-				if err == nil {
-					out = newOut
-				}
-				return false
-			}
-			return true
-		},
-		nil,
-	)
-	return out
-}
-
-// Build a cue instance from a directory and args
-func cueBuild(r *cue.Runtime, cueRoot string, buildArgs ...string) (*cue.Instance, error) {
-	var err error
-	cueRoot, err = filepath.Abs(cueRoot)
-	if err != nil {
-		return nil, err
-	}
-	buildConfig := &cueload.Config{
-		ModuleRoot: cueRoot,
-		Dir:        cueRoot,
-	}
-	instances := cueload.Instances(buildArgs, buildConfig)
-	if len(instances) != 1 {
-		return nil, errors.New("only one package is supported at a time")
-	}
-	return r.Build(instances[0])
-}
-
 func debugJSON(v interface{}) {
 	if os.Getenv("DEBUG") != "" {
 		e := json.NewEncoder(os.Stderr)
@@ -144,104 +58,6 @@ func randomID(size int) (string, error) {
 	return fmt.Sprintf("%x", b), nil
 }
 
-func cueWrapExpr(p string, v cueAst.Expr) (cueAst.Expr, error) {
-	pExpr, err := cueParser.ParseExpr("path", p)
-	if err != nil {
-		return v, err
-	}
-	out := v
-	cursor := pExpr
-walk:
-	for {
-		switch c := cursor.(type) {
-		case *cueAst.SelectorExpr:
-			out = cueAst.NewStruct(
-				&cueAst.Field{
-					Value: out,
-					Label: c.Sel,
-				},
-			)
-			cursor = c.X
-		case *cueAst.Ident:
-			out = cueAst.NewStruct(
-				&cueAst.Field{
-					Value: out,
-					Label: c,
-				},
-			)
-			break walk
-		default:
-			return out, fmt.Errorf("invalid path expression: %q", p)
-		}
-	}
-	return out, nil
-}
-
-func cueWrapFile(p string, v interface{}) (*cueAst.File, error) {
-	f, err := cueParser.ParseFile("value", v)
-	if err != nil {
-		return f, err
-	}
-	decls := make([]cueAst.Decl, 0, len(f.Decls))
-	for _, decl := range f.Decls {
-		switch d := decl.(type) {
-		case *cueAst.Field:
-			wrappedExpr, err := cueWrapExpr(p, cueAst.NewStruct(d))
-			if err != nil {
-				return f, err
-			}
-			decls = append(decls, &cueAst.EmbedDecl{Expr: wrappedExpr})
-		case *cueAst.EmbedDecl:
-			wrappedExpr, err := cueWrapExpr(p, d.Expr)
-			if err != nil {
-				return f, err
-			}
-			d.Expr = wrappedExpr
-			decls = append(decls, d)
-		case *cueAst.ImportDecl:
-			decls = append(decls, decl)
-		default:
-			fmt.Printf("skipping unsupported decl type %#v\n\n", decl)
-			continue
-		}
-	}
-	f.Decls = decls
-	return f, nil
-}
-
-func cueIsEmptyStruct(v cue.Value) bool {
-	if st, err := v.Struct(); err == nil {
-		if st.Len() == 0 {
-			return true
-		}
-	}
-	return false
-}
-
-// Return false if v is not concrete, or contains any
-// non-concrete fields or items.
-func cueIsConcrete(v cue.Value) bool {
-	// FIXME: use Value.Walk?
-	if it, err := v.Fields(); err == nil {
-		for it.Next() {
-			if !cueIsConcrete(it.Value()) {
-				return false
-			}
-		}
-		return true
-	}
-	if it, err := v.List(); err == nil {
-		for it.Next() {
-			if !cueIsConcrete(it.Value()) {
-				return false
-			}
-		}
-		return true
-	}
-	dv, _ := v.Default()
-	return v.IsConcrete() || dv.IsConcrete()
-}
-
 // LLB Helper to pull a Docker image + all its metadata
 func llbDockerImage(ref string) llb.State {
 	return llb.Image(
@@ -272,17 +88,3 @@ func cueCleanPath(p string) (string, error) {
 	cp := cue.ParsePath(p)
 	return cp.String(), cp.Err()
 }
-
-func autoMarshal(value interface{}) ([]byte, error) {
-	switch v := value.(type) {
-	case []byte:
-		return v, nil
-	case string:
-		return []byte(v), nil
-	case io.Reader:
-		return ioutil.ReadAll(v)
-	default:
-		return nil, fmt.Errorf("unsupported marshal inoput type")
-	}
-	return []byte(fmt.Sprintf("%v", value)), nil
-}

dagger/value.go

@@ -0,0 +1,341 @@
+package dagger
+
+import (
+	"fmt"
+	"os"
+
+	"cuelang.org/go/cue"
+	cueformat "cuelang.org/go/cue/format"
+	"github.com/moby/buildkit/client/llb"
+)
+
+// Polyfill for cue.Value.
+// Use instead of cue.Value and cue.Instance
+type Value struct {
+	// FIXME: don't embed, cleaner API
+	cue.Value
+	cc   *Compiler
+	inst *cue.Instance
+}
+
+func (v *Value) Lock() {
+	if v.cc == nil {
+		return
+	}
+	v.cc.Lock()
+}
+
+func (v *Value) Unlock() {
+	if v.cc == nil {
+		return
+	}
+	v.cc.Unlock()
+}
+
+func (v *Value) Lookup(path ...string) *Value {
+	v.Lock()
+	defer v.Unlock()
+	return v.Wrap(v.Unwrap().Lookup(path...))
+}
+
+func (v *Value) LookupPath(p cue.Path) *Value {
+	v.Lock()
+	defer v.Unlock()
+	return v.Wrap(v.Unwrap().LookupPath(p))
+}
+
+// FIXME: deprecated by Get()
+func (v *Value) LookupTarget(target string) *Value {
+	return v.LookupPath(cue.ParsePath(target))
+}
+
+func (v *Value) Get(target string) *Value {
+	return v.LookupPath(cue.ParsePath(target))
+}
+
+// Component returns the component value if v is a valid dagger component or an error otherwise.
+// If no '#dagger' annotation is present, os.ErrNotExist
+// is returned.
+func (v *Value) Component() (*Component, error) {
+	c := &Component{
+		v: v,
+	}
+	if !c.Exists() {
+		return c, os.ErrNotExist
+	}
+	if err := c.Validate(); err != nil {
+		return c, err
+	}
+	return c, nil
+}
+
+func (v *Value) Script() (*Script, error) {
+	s := &Script{
+		v: v,
+	}
+	if err := s.Validate(); err != nil {
+		return s, err
+	}
+	return s, nil
+}
+
+func (v *Value) Executable() (Executable, error) {
+	if script, err := v.Script(); err == nil {
+		return script, nil
+	}
+	if component, err := v.Component(); err == nil {
+		return component, nil
+	}
+	if op, err := v.Op(); err == nil {
+		return op, nil
+	}
+	return nil, fmt.Errorf("value is not executable")
+}
+
+// ScriptOrComponent returns one of:
+//  (1) the component value if v is a valid component (type *Component)
+//  (2) the script value if v is a valid script (type *Script)
+//  (3) an error otherwise
+func (v *Value) ScriptOrComponent() (interface{}, error) {
+	s, err := v.Script()
+	if err == nil {
+		return s, nil
+	}
+	c, err := v.Component()
+	if err == nil {
+		return c, nil
+	}
+	return nil, fmt.Errorf("not a script or component")
+}
+
+func (v *Value) Op() (*Op, error) {
+	// Merge #Op definition from spec to get default values
+	spec, err := v.Compiler().Spec()
+	if err != nil {
+		return nil, err
+	}
+	v, err = spec.Get("#Op").Merge(v)
+	if err != nil {
+		return nil, err
+	}
+	op := &Op{
+		v: v,
+	}
+	return op, nil
+}
+
+func (v *Value) Mount(dest string) (*Mount, error) {
+	mnt := &Mount{
+		v:    v,
+		dest: dest,
+	}
+	return mnt, mnt.Validate()
+}
+
+// Interpret this value as a spec
+func (v *Value) Spec() (*Spec, error) {
+	// Spec must be a struct
+	if _, err := v.Struct(); err != nil {
+		return nil, err
+	}
+	return &Spec{
+		root: v,
+	}, nil
+}
+
+// FIXME: receive string path?
+func (v *Value) Merge(x interface{}, path ...string) (*Value, error) {
+	if xval, ok := x.(*Value); ok {
+		if xval.Compiler() != v.Compiler() {
+			return nil, fmt.Errorf("can't merge values from different compilers")
+		}
+		x = xval.Unwrap()
+	}
+	result := v.Wrap(v.Unwrap().Fill(x, path...))
+	return result, result.Validate()
+}
+
+func (v *Value) MergePath(x interface{}, p cue.Path) (*Value, error) {
+	// FIXME: array indexes and defs are not supported,
+	//  they will be silently converted to regular fields.
+	//  eg.  `foo.#bar[0]` will become `foo["#bar"]["0"]`
+	return v.Merge(x, cuePathToStrings(p)...)
+}
+
+func (v *Value) MergeTarget(x interface{}, target string) (*Value, error) {
+	return v.MergePath(x, cue.ParsePath(target))
+}
+
+func (v *Value) RangeList(fn func(int, *Value) error) error {
+	it, err := v.List()
+	if err != nil {
+		return err
+	}
+	i := 0
+	for it.Next() {
+		if err := fn(i, v.Wrap(it.Value())); err != nil {
+			return err
+		}
+		i += 1
+	}
+	return nil
+}
+
+func (v *Value) RangeStruct(fn func(string, *Value) error) error {
+	it, err := v.Fields()
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		if err := fn(it.Label(), v.Wrap(it.Value())); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Recursive concreteness check.
+// Return false if v is not concrete, or contains any
+// non-concrete fields or items.
+func (v *Value) IsConcreteR() bool {
+	// FIXME: use Value.Walk
+	if it, err := v.Fields(); err == nil {
+		for it.Next() {
+			w := v.Wrap(it.Value())
+			if !w.IsConcreteR() {
+				return false
+			}
+		}
+		return true
+	}
+	if it, err := v.List(); err == nil {
+		for it.Next() {
+			w := v.Wrap(it.Value())
+			if !w.IsConcreteR() {
+				return false
+			}
+		}
+		return true
+	}
+	dv, _ := v.Default()
+	return v.IsConcrete() || dv.IsConcrete()
+}
+
+// Export concrete values to JSON. ignoring non-concrete values.
+// Contrast with cue.Value.MarshalJSON which requires all values
+// to be concrete.
+func (v *Value) JSON() JSON {
+	v.Lock()
+	defer v.Unlock()
+	var out JSON
+	v.Walk(
+		func(v cue.Value) bool {
+			b, err := v.MarshalJSON()
+			if err == nil {
+				newOut, err := out.Set(b, cuePathToStrings(v.Path())...)
+				if err == nil {
+					out = newOut
+				}
+				return false
+			}
+			return true
+		},
+		nil,
+	)
+	return out
+}
+
+func (v *Value) SaveJSON(fs FS, filename string) FS {
+	return fs.Change(func(st llb.State) llb.State {
+		return st.File(
+			llb.Mkfile(filename, 0600, v.JSON()),
+		)
+	})
+}
+
+func (v *Value) Save(fs FS, filename string) (FS, error) {
+	src, err := v.Source()
+	if err != nil {
+		return fs, err
+	}
+	return fs.Change(func(st llb.State) llb.State {
+		return st.File(
+			llb.Mkfile(filename, 0600, src),
+		)
+	}), nil
+}
+
+func (v *Value) Validate(defs ...string) error {
+	if err := v.Unwrap().Validate(); err != nil {
+		return err
+	}
+	if len(defs) == 0 {
+		return nil
+	}
+	spec, err := v.Compiler().Spec()
+	if err != nil {
+		return err
+	}
+	for _, def := range defs {
+		if err := spec.Validate(v, def); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Value implements Fillable.
+// This is the only method which changes the value in-place.
+// FIXME this co-exists awkwardly with the rest of Value.
+func (v *Value) Fill(x interface{}) error {
+	v.Value = v.Value.Fill(x)
+	return v.Validate()
+}
+
+func (v *Value) Source() ([]byte, error) {
+	v.Lock()
+	defer v.Unlock()
+	return cueformat.Node(v.Eval().Syntax())
+}
+
+func (v *Value) IsEmptyStruct() bool {
+	if st, err := v.Struct(); err == nil {
+		if st.Len() == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func (v *Value) CueInst() *cue.Instance {
+	return v.inst
+}
+
+func (v *Value) Compiler() *Compiler {
+	if v.cc == nil {
+		return &Compiler{}
+	}
+	return v.cc
+}
+
+func (v *Value) Debugf(msg string, args ...interface{}) {
+	prefix := v.Path().String()
+	args = append([]interface{}{prefix}, args...)
+	debugf("%s: "+msg, args...)
+}
+
+func (v *Value) Wrap(v2 cue.Value) *Value {
+	return wrapValue(v2, v.inst, v.cc)
+}
+
+func (v *Value) Unwrap() cue.Value {
+	return v.Value
+}
+
+func wrapValue(v cue.Value, inst *cue.Instance, cc *Compiler) *Value {
+	return &Value{
+		Value: v,
+		cc:    cc,
+		inst:  inst,
+	}
+}

dagger/value_test.go

@@ -0,0 +1,35 @@
+package dagger
+
+import (
+	"testing"
+)
+
+func TestSimple(t *testing.T) {
+	cc := &Compiler{}
+	_, err := cc.EmptyStruct()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCompileBootScript(t *testing.T) {
+	cc := &Compiler{}
+	s, err := cc.CompileScript("boot.cue", defaultBootScript)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Validate(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCompileSimpleScript(t *testing.T) {
+	cc := &Compiler{}
+	s, err := cc.CompileScript("simple.cue", `[{do: "local", dir: "."}]`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Validate(); err != nil {
+		t.Fatal(err)
+	}
+}

main.go

@@ -1,43 +0,0 @@
-// A simple main.go for testing the dagger Go API
-package main
-
-import (
-	"context"
-	"fmt"
-	"os"
-
-	"dagger.cloud/go/dagger"
-)
-
-func main() {
-	ctx := context.TODO()
-	c, err := dagger.NewClient(ctx, "")
-	if err != nil {
-		fatal(err)
-	}
-
-	configPath := "."
-	if len(os.Args) > 1 {
-		configPath = os.Args[1]
-	}
-
-	if err := c.SetConfig(configPath); err != nil {
-		fatal(err)
-	}
-
-	// if err := c.ConnectInput("source", os.Getenv("HOME")+"/Documents/github/samalba/hello-go"); err != nil {
-	// 	fatal(err)
-	// }
-	if err := c.Run(ctx, "compute"); err != nil {
-		fatal(err)
-	}
-}
-
-func fatalf(msg string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, msg, args...)
-	os.Exit(1)
-}
-
-func fatal(msg interface{}) {
-	fatalf("%s\n", msg)
-}