diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index cd98b3e4..817a3b24 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "dagger.cloud/go/dagger" "dagger.cloud/go/dagger/ui" @@ -16,32 +15,20 @@ var computeCmd = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { ctx := context.TODO() - c, err := dagger.NewClient(ctx, "") + // FIXME: boot and bootdir should be config fields, not args + c, err := dagger.NewClient(ctx, "", "", args[0]) if err != nil { ui.Fatal(err) } - target := args[0] - if target == "-" { - ui.Info("Assembling config from stdin\n") - // FIXME: include cue.mod/pkg from *somewhere* so stdin config can import - if err := c.SetConfig(os.Stdin); err != nil { - ui.Fatal(err) - } - } else { - ui.Info("Assembling config from %q\n", target) - if err := c.SetConfig(target); err != nil { - ui.Fatal(err) - } - } + // FIXME: configure which config to compute (duh) + // FIXME: configure inputs ui.Info("Running") - output, err := c.Run(ctx, "compute") + output, err := c.Compute(ctx) if err != nil { ui.Fatal(err) } ui.Info("Processing output") - // output.Print(os.Stdout) fmt.Println(output.JSON()) - // FIXME: write computed values back to env dir }, } diff --git a/dagger/client.go b/dagger/client.go index 5a48195c..eaf30e7e 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "strings" "time" @@ -15,14 +14,10 @@ import ( // Cue "cuelang.org/go/cue" - cueerrors "cuelang.org/go/cue/errors" - cueformat "cuelang.org/go/cue/format" // buildkit bk "github.com/moby/buildkit/client" _ "github.com/moby/buildkit/client/connhelper/dockercontainer" - "github.com/moby/buildkit/client/llb" - bkgw "github.com/moby/buildkit/frontend/gateway/client" // docker output "github.com/containerd/console" @@ -32,21 +27,34 @@ import ( const ( defaultBuildkitHost = "docker-container://buildkitd" - bkConfigKey = "context" - bkInputKey = ":dagger:input:" - bkActionKey = ":dagger:action:" + bkBootKey = "boot" + bkInputKey = "input" + + defaultBootDir = "." + + // FIXME: rename to defaultConfig ? + defaultBootScript = ` +bootdir: string | *"." +bootscript: [ + { + do: "local" + dir: bootdir + include: ["*.cue", "cue.mod"] + }, +] +` ) type Client struct { c *bk.Client - inputs map[string]llb.State localdirs map[string]string - - BKFrontend bkgw.BuildFunc + boot string + bootdir string + input string } -func NewClient(ctx context.Context, host string) (*Client, error) { +func NewClient(ctx context.Context, host, boot, bootdir string) (*Client, error) { // buildkit client if host == "" { host = os.Getenv("BUILDKIT_HOST") @@ -54,107 +62,59 @@ func NewClient(ctx context.Context, host string) (*Client, error) { if host == "" { host = defaultBuildkitHost } + if boot == "" { + boot = defaultBootScript + } + if bootdir == "" { + bootdir = defaultBootDir + } c, err := bk.New(ctx, host) if err != nil { return nil, errors.Wrap(err, "buildkit client") } return &Client{ c: c, - inputs: map[string]llb.State{}, + boot: boot, + bootdir: bootdir, + input: `{}`, localdirs: map[string]string{}, }, nil } -func (c *Client) ConnectInput(target string, input interface{}) error { - var st llb.State - switch in := input.(type) { - case llb.State: - st = in - case string: - // Generate a random local input label for security - st = c.AddLocalDir(in, target) - default: - return fmt.Errorf("unsupported input type") +func (c *Client) LocalDirs() ([]string, error) { + boot, err := c.BootScript() + if err != nil { + return nil, err } - c.inputs[bkInputKey+target] = st - return nil + return boot.LocalDirs() } -func (c *Client) AddLocalDir(dir, label string, opts ...llb.LocalOption) llb.State { - c.localdirs[label] = dir - return llb.Local(label, opts...) -} - -// Set cue config for future calls. -// input can be: -// - llb.State: valid cue config directory -// - io.Reader: valid cue source -// - string: local path to valid cue file or directory -// - func(llb.State)llb.Stte: modify existing state - -func (c *Client) SetConfig(inputs ...interface{}) error { - for _, input := range inputs { - if err := c.setConfig(input); err != nil { - return err - } +func (c *Client) BootScript() (*Script, error) { + debugf("compiling boot script: %q\n", c.boot) + cc := &Compiler{} + src, err := cc.Compile("boot.cue", c.boot) + if err != nil { + return nil, errors.Wrap(err, "compile") } - return nil -} - -func (c *Client) setConfig(input interface{}) error { - var st llb.State - switch in := input.(type) { - case llb.State: - st = in - case func(llb.State) llb.State: - // Modify previous state - last, ok := c.inputs[bkConfigKey] - if !ok { - last = llb.Scratch() - } - st = in(last) - case io.Reader: - contents, err := ioutil.ReadAll(in) - if err != nil { - return err - } - st = llb.Scratch().File(llb.Mkfile( - "config.cue", - 0660, - contents, - )) - // Interpret string as a path (dir or file) - case string: - info, err := os.Stat(in) - if err != nil { - return err - } - if info.IsDir() { - // FIXME: include pattern *.cue ooh yeah - st = c.AddLocalDir(in, "config", - //llb.IncludePatterns([]string{"*.cue", "cue.mod"})), - llb.FollowPaths([]string{"*.cue", "cue.mod"}), - ) - } else { - f, err := os.Open(in) - if err != nil { - return err - } - defer f.Close() - return c.SetConfig(f) - } + src, err = src.MergeTarget(c.bootdir, "bootdir") + if err != nil { + return nil, err } - c.inputs[bkConfigKey] = st - return nil + return src.Get("bootscript").Script() } -func (c *Client) Run(ctx context.Context, action string) (*Output, error) { +func (c *Client) Compute(ctx context.Context) (*Value, error) { + cc := &Compiler{} + out, err := cc.EmptyStruct() + if err != nil { + return nil, err + } // Spawn Build() goroutine eg, ctx := errgroup.WithContext(ctx) events := make(chan *bk.SolveStatus) outr, outw := io.Pipe() // Spawn build function - eg.Go(c.buildfn(ctx, action, events, outw)) + eg.Go(c.buildfn(ctx, events, outw)) // Spawn print function(s) dispCtx := context.TODO() var eventsdup chan *bk.SolveStatus @@ -164,21 +124,36 @@ func (c *Client) Run(ctx context.Context, action string) (*Output, error) { } eg.Go(c.printfn(dispCtx, events, eventsdup)) // Retrieve output - out := NewOutput() eg.Go(c.outputfn(ctx, outr, out)) return out, eg.Wait() } -func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveStatus, w io.WriteCloser) func() error { - return func() error { - defer debugf("buildfn complete") +func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.WriteCloser) func() error { + return func() (err error) { + defer func() { + debugf("buildfn complete, err=%q", err) + if err != nil { + // Close exporter pipe so that export processor can return + w.Close() + } + }() + boot, err := c.BootScript() + if err != nil { + close(ch) + return errors.Wrap(err, "assemble boot script") + } + bootSource, err := boot.Value().Source() + if err != nil { + close(ch) + return errors.Wrap(err, "serialize boot script") + } // Setup solve options opts := bk.SolveOpt{ FrontendAttrs: map[string]string{ - bkActionKey: action, + bkInputKey: c.input, + bkBootKey: string(bootSource), }, - LocalDirs: c.localdirs, - FrontendInputs: c.inputs, + LocalDirs: map[string]string{}, // FIXME: catch output & return as cue value Exports: []bk.ExportEntry{ { @@ -189,16 +164,18 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt }, }, } - // Setup frontend - bkFrontend := c.BKFrontend - if bkFrontend == nil { - r := &Runtime{} - bkFrontend = r.BKFrontend - } - resp, err := c.c.Build(ctx, opts, "", bkFrontend, ch) + // Connect local dirs + localdirs, err := c.LocalDirs() + if err != nil { + close(ch) + return errors.Wrap(err, "connect local dirs") + } + for _, dir := range localdirs { + opts.LocalDirs[dir] = dir + } + // Call buildkit solver + resp, err := c.c.Build(ctx, opts, "", Compute, ch) if err != nil { - // Close exporter pipe so that export processor can return - w.Close() err = errors.New(bkCleanError(err.Error())) return errors.Wrap(err, "buildkit solve") } @@ -211,7 +188,7 @@ func (c *Client) buildfn(ctx context.Context, action string, ch chan *bk.SolveSt } // Read tar export stream from buildkit Build(), and extract cue output -func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func() error { +func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Value) func() error { return func() error { defer debugf("outputfn complete") tr := tar.NewReader(r) @@ -229,14 +206,13 @@ func (c *Client) outputfn(ctx context.Context, r io.Reader, out *Output) func() continue } debugf("outputfn: compiling & merging %q", h.Name) - // FIXME: only doing this for debug. you can pass tr directly as io.Reader. - contents, err := ioutil.ReadAll(tr) + + cc := out.Compiler() + v, err := cc.Compile(h.Name, tr) if err != nil { return err } - //if err := out.FillSource(h.Name, tr); err != nil { - if err := out.FillSource(h.Name, contents); err != nil { - debugf("error with %s: contents=\n------\n%s\n-----\n", h.Name, contents) + if err := out.Fill(v); err != nil { return errors.Wrap(err, h.Name) } debugf("outputfn: DONE: compiling & merging %q", h.Name) @@ -327,7 +303,7 @@ func (c *Client) printfn(ctx context.Context, ch, ch2 chan *bk.SolveStatus) func for _, v := range status.Vertexes { p := cue.ParsePath(v.Name) if err := p.Err(); err != nil { - debugf("ignoring buildkit vertex %q: not a valid cue path", p.String()) + debugf("ignoring buildkit vertex %q: not a valid cue path", v.Name) continue } n := &Node{ @@ -377,58 +353,3 @@ func (c *Client) dockerprintfn(ctx context.Context, ch chan *bk.SolveStatus, out return progressui.DisplaySolveStatus(ctx, "", cons, out, ch) } } - -type Output struct { - r *cue.Runtime - inst *cue.Instance -} - -func NewOutput() *Output { - r := &cue.Runtime{} - inst, _ := r.Compile("", "") - return &Output{ - r: r, - inst: inst, - } -} - -func (o *Output) Print(w io.Writer) error { - v := o.Cue().Value().Eval() - b, err := cueformat.Node(v.Syntax()) - if err != nil { - return err - } - _, err = w.Write(b) - return err -} - -func (o *Output) JSON() JSON { - return cueToJSON(o.Cue().Value()) -} - -func (o *Output) Cue() *cue.Instance { - return o.inst -} - -func (o *Output) FillSource(filename string, x interface{}) error { - inst, err := o.r.Compile(filename, x) - if err != nil { - return fmt.Errorf("compile %s: %s", filename, cueerrors.Details(err, nil)) - } - if err := o.FillValue(inst.Value()); err != nil { - return fmt.Errorf("merge %s: %s", filename, cueerrors.Details(err, nil)) - } - return nil -} - -func (o *Output) FillValue(x interface{}) error { - inst, err := o.inst.Fill(x) - if err != nil { - return err - } - if err := inst.Value().Validate(); err != nil { - return err - } - o.inst = inst - return nil -} diff --git a/dagger/compiler.go b/dagger/compiler.go new file mode 100644 index 00000000..0abc300e --- /dev/null +++ b/dagger/compiler.go @@ -0,0 +1,113 @@ +//go:generate sh gen.sh +package dagger + +import ( + "context" + "path" + "path/filepath" + "sync" + + "cuelang.org/go/cue" + cueload "cuelang.org/go/cue/load" + "github.com/pkg/errors" +) + +// Polyfill for a cue runtime +// (we call it compiler to avoid confusion with dagger runtime) +// Use this instead of cue.Runtime +type Compiler struct { + sync.Mutex + cue.Runtime + spec *Spec +} + +func (cc *Compiler) Cue() *cue.Runtime { + return &(cc.Runtime) +} + +func (cc *Compiler) Spec() (*Spec, error) { + if cc.spec != nil { + return cc.spec, nil + } + v, err := cc.Compile("spec.cue", DaggerSpec) + if err != nil { + return nil, err + } + spec, err := v.Spec() + if err != nil { + return nil, err + } + cc.spec = spec + return spec, nil +} + +// Compile an empty struct +func (cc *Compiler) EmptyStruct() (*Value, error) { + return cc.Compile("", "") +} + +func (cc *Compiler) Compile(name string, src interface{}) (*Value, error) { + inst, err := cc.Cue().Compile(name, src) + if err != nil { + // FIXME: cleaner way to unwrap cue error details? + return nil, cueErr(err) + } + return cc.Wrap(inst.Value(), inst), nil +} + +func (cc *Compiler) CompileScript(name string, src interface{}) (*Script, error) { + v, err := cc.Compile(name, src) + if err != nil { + return nil, err + } + return v.Script() +} + +// Build a cue configuration tree from the files in fs. +func (cc *Compiler) Build(ctx context.Context, fs FS, args ...string) (*Value, error) { + debugf("Compiler.Build") + defer debugf("COMPLETE: Compiler.Build") + // The CUE overlay needs to be prefixed by a non-conflicting path with the + // local filesystem, otherwise Cue will merge the Overlay with whatever Cue + // files it finds locally. + const overlayPrefix = "/config" + + buildConfig := &cueload.Config{ + Dir: overlayPrefix, + Overlay: map[string]cueload.Source{}, + } + buildArgs := args + + err := fs.Walk(ctx, func(p string, f Stat) error { + debugf(" Compiler.Build: processing %q", p) + if f.IsDir() { + return nil + } + if filepath.Ext(p) != ".cue" { + return nil + } + contents, err := fs.ReadFile(ctx, p) + if err != nil { + return errors.Wrap(err, p) + } + overlayPath := path.Join(overlayPrefix, p) + buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents) + return nil + }) + if err != nil { + return nil, err + } + instances := cueload.Instances(buildArgs, buildConfig) + if len(instances) != 1 { + return nil, errors.New("only one package is supported at a time") + } + inst, err := cc.Cue().Build(instances[0]) + if err != nil { + return nil, err + } + return cc.Wrap(inst.Value(), inst), nil +} + +func (cc *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value { + return wrapValue(v, inst, cc) +} diff --git a/dagger/component.go b/dagger/component.go new file mode 100644 index 00000000..8735d97a --- /dev/null +++ b/dagger/component.go @@ -0,0 +1,66 @@ +package dagger + +import ( + "context" +) + +type Component struct { + v *Value +} + +func (c *Component) Value() *Value { + return c.v +} + +func (c *Component) Exists() bool { + // Does #dagger exist? + if c.Config().Err() != nil { + return false + } + return true +} + +// Return the contents of the "#dagger" annotation. +func (c *Component) Config() *Value { + return c.Value().Get("#dagger") +} + +// Verify that this component respects the dagger component spec. +// +// NOTE: calling matchSpec("#Component") is not enough because +// it does not match embedded scalars. +func (c Component) Validate() error { + return c.Config().Validate("#ComponentConfig") +} + +// Return this component's compute script. +func (c Component) ComputeScript() (*Script, error) { + return c.Value().Get("#dagger.compute").Script() +} + +// Compute the configuration for this component. +// Note that we simply execute the underlying compute script from an +// empty filesystem state. +// (It is never correct to pass an input filesystem state to compute a component) +func (c *Component) Compute(ctx context.Context, s Solver, out Fillable) (FS, error) { + return c.Execute(ctx, s.Scratch(), out) +} + +// A component implements the Executable interface by returning its +// compute script. +// See Value.Executable(). +func (c *Component) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) { + script, err := c.ComputeScript() + if err != nil { + return fs, err + } + return script.Execute(ctx, fs, out) +} + +func (c *Component) Walk(fn func(*Op) error) error { + script, err := c.ComputeScript() + if err != nil { + return err + } + return script.Walk(fn) +} diff --git a/dagger/compute.go b/dagger/compute.go new file mode 100644 index 00000000..f5ea6563 --- /dev/null +++ b/dagger/compute.go @@ -0,0 +1,41 @@ +package dagger + +import ( + "context" + "fmt" + + cueerrors "cuelang.org/go/cue/errors" + bkgw "github.com/moby/buildkit/frontend/gateway/client" +) + +// Buildkit compute entrypoint (BK calls if "solve" or "build") +// Use by wrapping in a buildkit client Build call, or buildkit frontend. +func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) { + // FIXME: wrap errors to avoid crashing buildkit Build() + // with cue error types (why??) + defer func() { + if err != nil { + err = fmt.Errorf("%s", cueerrors.Details(err, nil)) + debugf("execute returned an error. Wrapping...") + } + }() + debugf("initializing env") + env, err := NewEnv(ctx, c) + if err != nil { + return nil, err + } + debugf("computing env") + // Compute output overlay + if err := env.Compute(ctx); err != nil { + return nil, err + } + debugf("exporting env") + // Export env to a cue directory + outdir := NewSolver(c).Scratch() + outdir, err = env.Export(outdir) + if err != nil { + return nil, err + } + // Wrap cue directory in buildkit result + return outdir.Result(ctx) +} diff --git a/dagger/env.go b/dagger/env.go new file mode 100644 index 00000000..0952939d --- /dev/null +++ b/dagger/env.go @@ -0,0 +1,186 @@ +package dagger + +import ( + "context" + "os" + + "cuelang.org/go/cue" + cueflow "cuelang.org/go/tools/flow" + bkgw "github.com/moby/buildkit/frontend/gateway/client" + "github.com/pkg/errors" +) + +type Env struct { + // Base config + base *Value + // Input overlay: user settings, external directories, secrets... + input *Value + + // Output overlay: computed values, generated directories + output *Value + + // Buildkit solver + s Solver + + // shared cue compiler + // (because cue API requires shared runtime for everything) + cc *Compiler +} + +// Initialize a new environment +func NewEnv(ctx context.Context, c bkgw.Client) (*Env, error) { + cc := &Compiler{} + // 1. Load base config (specified by client) + debugf("Loading base configuration") + base, err := envBase(ctx, c, cc) + if err != nil { + return nil, err + } + // 2. Load input overlay (specified by client) + debugf("Loading input overlay") + input, err := envInput(ctx, c, cc) + if err != nil { + return nil, err + } + if _, err := base.Merge(input); err != nil { + return nil, errors.Wrap(err, "merge base & input") + } + return &Env{ + base: base, + input: input, + s: NewSolver(c), + cc: cc, + }, nil +} + +// Compute missing values in env configuration, and write them to state. +func (env *Env) Compute(ctx context.Context) error { + debugf("Computing environment") + output, err := env.Walk(ctx, func(c *Component, out Fillable) error { + _, err := c.Compute(ctx, env.s, out) + return err + }) + if err != nil { + return err + } + env.output = output + return nil +} + +// Export env to a directory of cue files +// (Use with FS.Change) +func (env *Env) Export(fs FS) (FS, error) { + // FIXME: we serialize as JSON to guarantee a self-contained file. + // Value.Save() leaks imports, so requires a shared cue.mod with + // client which is undesirable. + // Once Value.Save() resolves non-builtin imports with a tree shake, + // we can use it here. + fs = env.base.SaveJSON(fs, "base.cue") + fs = env.input.SaveJSON(fs, "input.cue") + if env.output != nil { + fs = env.output.SaveJSON(fs, "output.cue") + } + return fs, nil +} + +type EnvWalkFunc func(*Component, Fillable) error + +// Walk components and return any computed values +func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) { + debugf("Env.Walk") + defer debugf("COMPLETE: Env.Walk") + // Cueflow cue instance + // FIXME: make this cleaner in *Value by keeping intermediary instances + flowInst, err := env.base.CueInst().Fill(env.input.CueInst()) + if err != nil { + return nil, err + } + // Initialize empty output + out, err := env.cc.EmptyStruct() + if err != nil { + return nil, err + } + // Cueflow config + flowCfg := &cueflow.Config{ + UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error { + debugf("compute step") + if t == nil { + return nil + } + debugf("cueflow task %q: %s", t.Path().String(), t.State().String()) + if t.State() != cueflow.Terminated { + return nil + } + debugf("cueflow task %q: filling result", t.Path().String()) + // Merge task value into output + var err error + // FIXME: does cueflow.Task.Value() contain only filled values, + // or base + filled? + out, err = out.MergePath(t.Value(), t.Path()) + if err != nil { + return err + } + return nil + }, + } + // Cueflow match func + flowMatchFn := func(v cue.Value) (cueflow.Runner, error) { + val := env.cc.Wrap(v, flowInst) + c, err := val.Component() + if os.IsNotExist(err) { + // Not a component: skip + return nil, nil + } + if err != nil { + return nil, err + } + return cueflow.RunnerFunc(func(t *cueflow.Task) error { + return fn(c, t) + }), nil + } + // Orchestrate execution with cueflow + flow := cueflow.New(flowCfg, flowInst, flowMatchFn) + if err := flow.Run(ctx); err != nil { + return out, err + } + return out, nil +} + +func envBase(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) { + // 1. Receive boot script from client. + debugf("retrieving boot script") + bootSrc, exists := c.BuildOpts().Opts["boot"] + if !exists { + // No boot script: return empty base config + return cc.EmptyStruct() + } + + // 2. Compile & execute boot script + debugf("compiling boot script") + boot, err := cc.CompileScript("boot.cue", bootSrc) + if err != nil { + return nil, errors.Wrap(err, "compile boot script") + } + debugf("executing boot script") + bootState, err := boot.Execute(ctx, NewSolver(c).Scratch(), Discard()) + if err != nil { + return nil, errors.Wrap(err, "execute boot script") + } + // 3. load cue files produced by bootstrap script + // FIXME: BuildAll() to force all files (no required package..) + debugf("building cue configuration from boot state") + base, err := cc.Build(ctx, bootState) + debugf("done building cue configuration: err=%q", err) + return base, err +} + +func envInput(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) { + // 1. Receive input overlay from client. + // This is used to provide run-time settings, directories.. + inputSrc, exists := c.BuildOpts().Opts["input"] + if !exists { + // No input overlay: return empty tree + return cc.EmptyStruct() + } + return cc.Compile("input.cue", inputSrc) +} diff --git a/dagger/fs.go b/dagger/fs.go new file mode 100644 index 00000000..7cec80d9 --- /dev/null +++ b/dagger/fs.go @@ -0,0 +1,143 @@ +package dagger + +import ( + "context" + "path" + + "github.com/moby/buildkit/client/llb" + bkgw "github.com/moby/buildkit/frontend/gateway/client" + fstypes "github.com/tonistiigi/fsutil/types" +) + +type Stat struct { + *fstypes.Stat +} + +type FS struct { + // Before last solve + input llb.State + // After last solve + output bkgw.Reference + // How to produce the output + s Solver +} + +func (fs FS) Solver() Solver { + return fs.s +} + +// Compute output from input, if not done already. +// This method uses a pointer receiver to simplify +// calling it, since it is called in almost every +// other method. +func (fs *FS) solve(ctx context.Context) error { + if fs.output != nil { + return nil + } + output, err := fs.s.Solve(ctx, fs.input) + if err != nil { + return err + } + fs.output = output + return nil +} + +func (fs FS) ReadFile(ctx context.Context, filename string) ([]byte, error) { + // Lazy solve + if err := (&fs).solve(ctx); err != nil { + return nil, err + } + return fs.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename}) +} + +func (fs FS) ReadDir(ctx context.Context, dir string) ([]Stat, error) { + // Lazy solve + if err := (&fs).solve(ctx); err != nil { + return nil, err + } + st, err := fs.output.ReadDir(ctx, bkgw.ReadDirRequest{ + Path: dir, + }) + if err != nil { + return nil, err + } + out := make([]Stat, len(st)) + for i := range st { + out[i] = Stat{ + Stat: st[i], + } + } + return out, nil +} + +func (fs FS) walk(ctx context.Context, p string, fn WalkFunc) error { + files, err := fs.ReadDir(ctx, p) + if err != nil { + return err + } + for _, f := range files { + fPath := path.Join(p, f.GetPath()) + if err := fn(fPath, f); err != nil { + return err + } + if f.IsDir() { + if err := fs.walk(ctx, fPath, fn); err != nil { + return err + } + } + } + return nil +} + +type WalkFunc func(string, Stat) error + +func (fs FS) Walk(ctx context.Context, fn WalkFunc) error { + // Lazy solve + if err := (&fs).solve(ctx); err != nil { + return err + } + return fs.walk(ctx, "/", fn) +} + +type ChangeFunc func(llb.State) llb.State + +func (fs FS) Change(changes ...ChangeFunc) FS { + for _, change := range changes { + fs = fs.Set(change(fs.input)) + } + return fs +} + +func (fs FS) Set(st llb.State) FS { + fs.input = st + fs.output = nil + return fs +} + +func (fs FS) Solve(ctx context.Context) (FS, error) { + if err := (&fs).solve(ctx); err != nil { + return fs, err + } + return fs, nil +} + +func (fs FS) LLB() llb.State { + return fs.input +} + +func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) { + if err := (&fs).solve(ctx); err != nil { + return nil, err + } + return fs.output, nil +} + +func (fs FS) Result(ctx context.Context) (*bkgw.Result, error) { + res := bkgw.NewResult() + ref, err := fs.Ref(ctx) + if err != nil { + return nil, err + } + res.SetRef(ref) + return res, nil +} diff --git a/dagger/gen.go b/dagger/gen.go index 40f876c3..524aa628 100644 --- a/dagger/gen.go +++ b/dagger/gen.go @@ -46,6 +46,7 @@ package dagger // The contents of a #dagger annotation #ComponentConfig: { + // FIXME: deprecated input?: bool // script to compute the value @@ -91,7 +92,7 @@ package dagger #Script: [...#Op] // One operation in a script -#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy +#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy // Export a value from fs state to cue #Export: { @@ -101,22 +102,24 @@ package dagger format: "json"|"yaml"|*"string"|"number"|"boolean" } -#Load: #LoadComponent| #LoadScript -#LoadComponent: { - do: "load" - from: #Component -} -#LoadScript: { - do: "load" - from: #Script +#Local: { + do: "local" + dir: string + include: [...string] | *[] } +// FIXME: bring back load (more efficient than copy) + +#Load: { + do: "load" + from: #Component | #Script +} #Exec: { do: "exec" args: [...string] - env: [string]: string - always: true | *false + env?: [string]: string + always?: true | *false dir: string | *"/" mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript } @@ -146,11 +149,10 @@ package dagger #Copy: { do: "copy" from: #Script | #Component - src: string | *"/" - dest: string | *"/" + src?: string | *"/" + dest?: string | *"/" } - #TestScript: #Script & [ { do: "fetch-container", ref: "alpine:latest" }, { do: "exec", args: ["echo", "hello", "world" ] } diff --git a/dagger/job.go b/dagger/job.go deleted file mode 100644 index ee97eb03..00000000 --- a/dagger/job.go +++ /dev/null @@ -1,326 +0,0 @@ -package dagger - -import ( - "context" - "fmt" - "path" - "path/filepath" - "strings" - - "cuelang.org/go/cue" - cueerrors "cuelang.org/go/cue/errors" - cueload "cuelang.org/go/cue/load" - cueflow "cuelang.org/go/tools/flow" - "github.com/moby/buildkit/client/llb" - bkgw "github.com/moby/buildkit/frontend/gateway/client" - "github.com/pkg/errors" -) - -type Solver interface { - Solve(context.Context, llb.State) (bkgw.Reference, error) -} - -// 1 buildkit build = 1 job -type Job struct { - c bkgw.Client - // needed for cue operations - r *Runtime -} - -// Execute and wrap the result in a buildkit result -func (job Job) BKExecute(ctx context.Context) (_r *bkgw.Result, _e error) { - debugf("Executing bk frontend") - // wrap errors to avoid crashing buildkit with cue error types (why??) - defer func() { - if _e != nil { - _e = fmt.Errorf("%s", cueerrors.Details(_e, nil)) - debugf("execute returned an error. Wrapping...") - } - }() - out, err := job.Execute(ctx) - if err != nil { - return nil, err - } - // encode job output to buildkit result - debugf("[runtime] job executed. Encoding output") - // FIXME: we can't serialize result to standalone cue (with no imports). - // So the client cannot safely compile output without access to the same cue.mod - // as the runtime (which we don't want). - // So for now we return the output as json, still parsed as cue on the client - // to keep our options open. Once there is a "tree shake" primitive, we can - // use that to return cue. - // - // Uncomment to return actual cue: - // ---- - // outbytes, err := cueformat.Node(out.Value().Eval().Syntax()) - // if err != nil { - // return nil, err - // } - // ---- - outbytes := cueToJSON(out.Value()) - debugf("[runtime] output encoded. Writing output to exporter") - outref, err := job.Solve(ctx, - llb.Scratch().File(llb.Mkfile("computed.cue", 0600, outbytes)), - ) - if err != nil { - return nil, err - } - debugf("[runtime] output written to exporter. returning to buildkit solver") - res := bkgw.NewResult() - res.SetRef(outref) - return res, nil -} - -func (job Job) Execute(ctx context.Context) (_i *cue.Instance, _e error) { - debugf("[runtime] Execute()") - defer func() { debugf("[runtime] DONE Execute(): err=%v", _e) }() - state, err := job.Config(ctx) - if err != nil { - return nil, err - } - // Merge input information into the cue config - inputs, err := job.Inputs(ctx) - if err != nil { - return nil, err - } - for target := range inputs { - // FIXME: cleaner code generation, missing cue.Value.FillPath - state, err = job.r.fill(state, `#dagger: input: true`, target) - if err != nil { - return nil, errors.Wrapf(err, "connect input %q", target) - } - } - action := job.Action() - switch action { - case "compute": - return job.doCompute(ctx, state) - case "export": - return job.doExport(ctx, state) - default: - return job.doExport(ctx, state) - } -} - -func (job Job) doExport(ctx context.Context, state *cue.Instance) (*cue.Instance, error) { - return state, nil -} - -func (job Job) doCompute(ctx context.Context, state *cue.Instance) (*cue.Instance, error) { - out, err := job.r.Compile("computed.cue", "") - if err != nil { - return nil, err - } - // Setup cueflow - debugf("Setting up cueflow") - flow := cueflow.New( - &cueflow.Config{ - UpdateFunc: func(c *cueflow.Controller, t *cueflow.Task) error { - debugf("cueflow event") - if t == nil { - return nil - } - debugf("cueflow task %q: %s", t.Path().String(), t.State().String()) - if t.State() == cueflow.Terminated { - debugf("cueflow task %q: filling result", t.Path().String()) - out, err = out.Fill(t.Value(), cuePathToStrings(t.Path())...) - if err != nil { - return err - } - // FIXME: catch merge errors early with state - } - return nil - }, - }, - state, - // Task match func - func(v cue.Value) (cueflow.Runner, error) { - // Is v a component (has #dagger) with a field 'compute' ? - isComponent, err := job.r.isComponent(v, "compute") - if err != nil { - return nil, err - } - if !isComponent { - return nil, nil - } - debugf("[%s] component detected\n", v.Path().String()) - // task runner func - runner := cueflow.RunnerFunc(func(t *cueflow.Task) error { - computeScript := t.Value().LookupPath(cue.ParsePath("#dagger.compute")) - script, err := job.newScript(computeScript) - if err != nil { - return err - } - // Run the script & fill the result into the task - return script.Run(ctx, t) - }) - return runner, nil - }, - ) - debugf("Running cueflow") - if err := flow.Run(ctx); err != nil { - return nil, err - } - debugf("Completed cueflow run. Merging result.") - state, err = state.Fill(out) - if err != nil { - return nil, err - } - debugf("Result merged") - // Return only the computed values - return out, nil -} - -func (job Job) bk() bkgw.Client { - return job.c -} - -func (job Job) Action() string { - opts := job.bk().BuildOpts().Opts - if action, ok := opts[bkActionKey]; ok { - return action - } - return "" -} - -// Load the cue config for this job -// (received as llb input) -func (job Job) Config(ctx context.Context) (*cue.Instance, error) { - src := llb.Local(bkConfigKey, - llb.SessionID(job.bk().BuildOpts().SessionID), - llb.SharedKeyHint(bkConfigKey), - llb.WithCustomName("load config"), - ) - - bkInputs, err := job.bk().Inputs(ctx) - if err != nil { - return nil, err - } - if st, ok := bkInputs[bkConfigKey]; ok { - src = st - } - // job.runDebug(ctx, src, "ls", "-la", "/mnt") - return job.LoadCue(ctx, src) -} - -func (job Job) runDebug(ctx context.Context, mnt llb.State, args ...string) error { - opts := []llb.RunOption{ - llb.Args(args), - llb.AddMount("/mnt", mnt), - } - cmd := llb.Image("alpine").Run(opts...).Root() - ref, err := job.Solve(ctx, cmd) - if err != nil { - return errors.Wrap(err, "debug") - } - // force non-lazy solve - if _, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{Path: "/"}); err != nil { - return errors.Wrap(err, "debug") - } - return nil -} - -func (job Job) Inputs(ctx context.Context) (map[string]llb.State, error) { - bkInputs, err := job.bk().Inputs(ctx) - if err != nil { - return nil, err - } - inputs := map[string]llb.State{} - for key, input := range bkInputs { - if !strings.HasPrefix(key, bkInputKey) { - continue - } - target := strings.Replace(key, bkInputKey, "", 1) - targetPath := cue.ParsePath(target) - if err := targetPath.Err(); err != nil { - return nil, errors.Wrapf(err, "input target %q", target) - } - // FIXME: check that the path can be passed to Fill - // (eg. only regular fields, no array indexes, no defs) - // see cuePathToStrings - inputs[target] = input - } - return inputs, nil -} - -// loadFiles recursively loads all .cue files from a buildkit gateway -// FIXME: this is highly inefficient. -func loadFiles(ctx context.Context, ref bkgw.Reference, p, overlayPrefix string, overlay map[string]cueload.Source) error { - // FIXME: we cannot use `IncludePattern` here, otherwise sub directories - // (e.g. "cue.mod") will be skipped. - files, err := ref.ReadDir(ctx, bkgw.ReadDirRequest{ - Path: p, - }) - if err != nil { - return err - } - for _, f := range files { - fPath := path.Join(p, f.GetPath()) - if f.IsDir() { - if err := loadFiles(ctx, ref, fPath, overlayPrefix, overlay); err != nil { - return err - } - continue - } - - if filepath.Ext(fPath) != ".cue" { - continue - } - - contents, err := ref.ReadFile(ctx, bkgw.ReadRequest{ - Filename: fPath, - }) - if err != nil { - return errors.Wrap(err, f.GetPath()) - } - overlay[path.Join(overlayPrefix, fPath)] = cueload.FromBytes(contents) - } - - return nil -} - -func (job Job) LoadCue(ctx context.Context, st llb.State, args ...string) (*cue.Instance, error) { - // The CUE overlay needs to be prefixed by a non-conflicting path with the - // local filesystem, otherwise Cue will merge the Overlay with whatever Cue - // files it finds locally. - const overlayPrefix = "/config" - - buildConfig := &cueload.Config{ - Dir: overlayPrefix, - Overlay: map[string]cueload.Source{}, - } - buildArgs := args - - // Inject cue files from llb state into overlay - ref, err := job.Solve(ctx, st) - if err != nil { - return nil, err - } - if err := loadFiles(ctx, ref, ".", overlayPrefix, buildConfig.Overlay); err != nil { - return nil, err - } - - instances := cueload.Instances(buildArgs, buildConfig) - if len(instances) != 1 { - return nil, errors.New("only one package is supported at a time") - } - inst, err := job.r.Build(instances[0]) - if err != nil { - return nil, cueErr(err) - } - return inst, nil -} - -func (job Job) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) { - // marshal llb - def, err := st.Marshal(ctx, llb.LinuxAmd64) - if err != nil { - return nil, err - } - // call solve - res, err := job.bk().Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()}) - if err != nil { - return nil, err - } - // always use single reference (ignore multiple outputs & metadata) - return res.SingleRef() -} diff --git a/dagger/mount.go b/dagger/mount.go new file mode 100644 index 00000000..b66144f4 --- /dev/null +++ b/dagger/mount.go @@ -0,0 +1,38 @@ +package dagger + +import ( + "context" + "fmt" + + "github.com/moby/buildkit/client/llb" + "github.com/pkg/errors" +) + +type Mount struct { + dest string + v *Value +} + +func (mnt *Mount) Validate(defs ...string) error { + return mnt.v.Validate(append(defs, "#Mount")...) +} + +func (mnt *Mount) LLB(ctx context.Context, s Solver) (llb.RunOption, error) { + if err := mnt.Validate("#MountTmp"); err == nil { + return llb.AddMount(mnt.dest, llb.Scratch(), llb.Tmpfs()), nil + } + if err := mnt.Validate("#MountCache"); err == nil { + // FIXME: cache mount + return nil, fmt.Errorf("FIXME: cache mount not yet implemented") + } + // Compute source component or script, discarding fs writes & output value + from, err := mnt.v.Lookup("from").Executable() + if err != nil { + return nil, errors.Wrap(err, "from") + } + fromFS, err := from.Execute(ctx, s.Scratch(), Discard()) + if err != nil { + return nil, err + } + return llb.AddMount(mnt.dest, fromFS.LLB()), nil +} diff --git a/dagger/op.go b/dagger/op.go new file mode 100644 index 00000000..bef6e6df --- /dev/null +++ b/dagger/op.go @@ -0,0 +1,256 @@ +package dagger + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/moby/buildkit/client/llb" + "github.com/pkg/errors" +) + +type Op struct { + v *Value +} + +func (op *Op) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) { + action, err := op.Action() + if err != nil { + return fs, err + } + return action(ctx, fs, out) +} + +func (op *Op) Walk(fn func(*Op) error) error { + debugf("Walk %v", op.v) + isCopy := (op.Validate("#Copy") == nil) + isLoad := (op.Validate("#Load") == nil) + if isCopy || isLoad { + debugf("MATCH %v", op.v) + if from, err := op.Get("from").Executable(); err == nil { + if err := from.Walk(fn); err != nil { + return err + } + } + // FIXME: we tolerate "from" which is not executable + } + if err := op.Validate("#Exec"); err == nil { + return op.Get("mount").RangeStruct(func(k string, v *Value) error { + if from, err := op.Get("from").Executable(); err == nil { + if err := from.Walk(fn); err != nil { + return err + } + } + return nil + }) + } + // depth first + return fn(op) +} + +type Action func(context.Context, FS, Fillable) (FS, error) + +func (op *Op) Action() (Action, error) { + // An empty struct is allowed as a no-op, to be + // more tolerant of if() in arrays. + if op.v.IsEmptyStruct() { + return op.Nothing, nil + } + // Manually check for a 'do' field. + // This is necessary because Runtime.ValidateSpec has a bug + // where an empty struct matches everything. + // see https://github.com/cuelang/cue/issues/566#issuecomment-749735878 + // Once the bug is fixed, the manual check can be removed. + if _, err := op.Get("do").String(); err != nil { + return nil, fmt.Errorf("invalid action: no \"do\" field") + } + actions := map[string]Action{ + "#Copy": op.Copy, + "#Exec": op.Exec, + "#Export": op.Export, + "#FetchContainer": op.FetchContainer, + "#FetchGit": op.FetchGit, + "#Local": op.Local, + } + for def, action := range actions { + if err := op.Validate(def); err == nil { + op.v.Debugf("OP MATCH: %s", def) + return action, nil + } + } + return nil, fmt.Errorf("invalid operation: %s", op.v.JSON()) +} + +func (op *Op) Validate(defs ...string) error { + defs = append(defs, "#Op") + return op.v.Validate(defs...) +} + +func (op *Op) Copy(ctx context.Context, fs FS, out Fillable) (FS, error) { + // Decode copy options + src, err := op.Get("src").String() + if err != nil { + return fs, err + } + dest, err := op.Get("dest").String() + if err != nil { + return fs, err + } + from, err := op.Get("from").Executable() + if err != nil { + return fs, errors.Wrap(err, "from") + } + // Compute source component or script, discarding fs writes & output value + fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard()) + if err != nil { + return fs, err + } + return fs.Change(func(st llb.State) llb.State { + return st.File(llb.Copy( + fromFS.LLB(), + src, + dest, + // FIXME: allow more configurable llb options + // For now we define the following convenience presets: + &llb.CopyInfo{ + CopyDirContentsOnly: true, + CreateDestPath: true, + AllowWildcard: true, + }, + )) + }), nil // lazy solve +} + +func (op *Op) Nothing(ctx context.Context, fs FS, out Fillable) (FS, error) { + return fs, nil +} +func (op *Op) Local(ctx context.Context, fs FS, out Fillable) (FS, error) { + dir, err := op.Get("dir").String() + if err != nil { + return fs, err + } + var include []string + if err := op.Get("include").Decode(&include); err != nil { + return fs, err + } + return fs.Set(llb.Local(dir, llb.FollowPaths(include))), nil // lazy solve +} + +func (op *Op) Exec(ctx context.Context, fs FS, out Fillable) (FS, error) { + var opts []llb.RunOption + var cmd struct { + Args []string + Env map[string]string + Dir string + Always bool + } + op.v.Decode(&cmd) + // marker for status events + // FIXME + opts = append(opts, llb.WithCustomName(op.v.Path().String())) + // args + opts = append(opts, llb.Args(cmd.Args)) + // dir + dir := cmd.Dir + if dir == "" { + dir = "/" + } + // env + for k, v := range cmd.Env { + opts = append(opts, llb.AddEnv(k, v)) + } + // always? + if cmd.Always { + cacheBuster, err := randomID(8) + if err != nil { + return fs, err + } + opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster)) + } + // mounts + if err := op.v.RangeStruct(func(k string, v *Value) error { + mnt, err := v.Mount(k) + if err != nil { + return err + } + opt, err := mnt.LLB(ctx, fs.Solver()) + if err != nil { + return err + } + opts = append(opts, opt) + return nil + }); err != nil { + return fs, err + } + // --> Execute + return fs.Change(func(st llb.State) llb.State { + return st.Run(opts...).Root() + }), nil // lazy solve +} + +func (op *Op) Export(ctx context.Context, fs FS, out Fillable) (FS, error) { + source, err := op.Get("source").String() + if err != nil { + return fs, err + } + format, err := op.Get("format").String() + if err != nil { + return fs, err + } + contents, err := fs.ReadFile(ctx, source) + if err != nil { + return fs, errors.Wrapf(err, "export %s", source) + } + switch format { + case "string": + if err := out.Fill(string(contents)); err != nil { + return fs, err + } + case "json": + var o interface{} + if err := json.Unmarshal(contents, &o); err != nil { + return fs, err + } + if err := out.Fill(o); err != nil { + return fs, err + } + default: + return fs, fmt.Errorf("unsupported export format: %q", format) + } + return fs, nil +} + +func (op *Op) Load(ctx context.Context, fs FS, out Fillable) (FS, error) { + from, err := op.Get("from").Executable() + if err != nil { + return fs, errors.Wrap(err, "load") + } + fromFS, err := from.Execute(ctx, fs.Solver().Scratch(), Discard()) + if err != nil { + return fs, errors.Wrap(err, "load: compute source") + } + return fs.Set(fromFS.LLB()), nil +} + +func (op *Op) FetchContainer(ctx context.Context, fs FS, out Fillable) (FS, error) { + ref, err := op.Get("ref").String() + if err != nil { + return fs, err + } + return fs.Set(llb.Image(ref)), nil +} +func (op *Op) FetchGit(ctx context.Context, fs FS, out Fillable) (FS, error) { + remote, err := op.Get("remote").String() + if err != nil { + return fs, err + } + ref, err := op.Get("ref").String() + if err != nil { + return fs, err + } + return fs.Set(llb.Git(remote, ref)), nil // lazy solve +} + +func (op *Op) Get(target string) *Value { + return op.v.Get(target) +} diff --git a/dagger/op_test.go b/dagger/op_test.go new file mode 100644 index 00000000..d1ffa56d --- /dev/null +++ b/dagger/op_test.go @@ -0,0 +1,32 @@ +package dagger + +import ( + "testing" +) + +func TestCopyMatch(t *testing.T) { + cc := &Compiler{} + src := `do: "copy", from: [{do: "local", dir: "foo"}]` + v, err := cc.Compile("", src) + if err != nil { + t.Fatal(err) + } + op, err := v.Op() + if err != nil { + t.Fatal(err) + } + if err := op.Validate("#Copy"); err != nil { + t.Fatal(err) + } + n := 0 + err = op.Walk(func(op *Op) error { + n += 1 + return nil + }) + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Fatal(n) + } +} diff --git a/dagger/runtime.go b/dagger/runtime.go deleted file mode 100644 index bad7eb92..00000000 --- a/dagger/runtime.go +++ /dev/null @@ -1,109 +0,0 @@ -//go:generate sh gen.sh -package dagger - -import ( - "context" - "fmt" - "sync" - - "cuelang.org/go/cue" - cueerrors "cuelang.org/go/cue/errors" - bkgw "github.com/moby/buildkit/frontend/gateway/client" - "github.com/pkg/errors" -) - -type Runtime struct { - l sync.Mutex - - cue.Runtime -} - -func (r *Runtime) Cue() *cue.Runtime { - return &(r.Runtime) -} - -func (r *Runtime) fill(inst *cue.Instance, v interface{}, target string) (*cue.Instance, error) { - targetPath := cue.ParsePath(target) - if err := targetPath.Err(); err != nil { - return nil, err - } - p := cuePathToStrings(targetPath) - if src, ok := v.(string); ok { - vinst, err := r.Compile(target, src) - if err != nil { - return nil, err - } - return inst.Fill(vinst.Value(), p...) - } - return inst.Fill(v, p...) -} - -// func (r Runtime) Run(...) -// Buildkit run entrypoint -func (r *Runtime) BKFrontend(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { - return r.newJob(c).BKExecute(ctx) -} - -func (r *Runtime) newJob(c bkgw.Client) Job { - return Job{ - r: r, - c: c, - } -} - -// Check whether a value is a valid component -// FIXME: calling matchSpec("#Component") is not enough because -// it does not match embedded scalars. -func (r *Runtime) isComponent(v cue.Value, fields ...string) (bool, error) { - cfg := v.LookupPath(cue.ParsePath("#dagger")) - if cfg.Err() != nil { - // No "#dagger" -> not a component - return false, nil - } - for _, field := range fields { - if cfg.Lookup(field).Err() != nil { - return false, nil - } - } - if err := r.validateSpec(cfg, "#ComponentConfig"); err != nil { - return true, errors.Wrap(err, "invalid #dagger") - } - return true, nil -} - -// eg. validateSpec(op, "#Op") -// eg. validateSpec(dag, "#DAG") -func (r *Runtime) validateSpec(v cue.Value, defpath string) (err error) { - // Expand cue errors to get full details - // FIXME: there is probably a cleaner way to do this. - defer func() { - if err != nil { - err = fmt.Errorf("%s", cueerrors.Details(err, nil)) - } - }() - r.l.Lock() - defer r.l.Unlock() - - // FIXME cache spec instance - spec, err := r.Compile("dagger.cue", DaggerSpec) - if err != nil { - panic("invalid spec") - } - def := spec.Value().LookupPath(cue.ParsePath(defpath)) - if err := def.Err(); err != nil { - return err - } - v = v.Eval() - if err := v.Validate(); err != nil { - return err - } - res := def.Unify(v) - if err := res.Validate(cue.Final()); err != nil { - return err - } - return nil -} - -func (r *Runtime) matchSpec(v cue.Value, def string) bool { - return r.validateSpec(v, def) == nil -} diff --git a/dagger/script.go b/dagger/script.go index e5636a19..2b56843b 100644 --- a/dagger/script.go +++ b/dagger/script.go @@ -2,325 +2,69 @@ package dagger import ( "context" - "encoding/json" - "fmt" - "cuelang.org/go/cue" - "github.com/moby/buildkit/client/llb" "github.com/pkg/errors" ) type Script struct { - v cue.Value - job Job - - // current state - state *State + v *Value } -func (job Job) newScript(v cue.Value) (*Script, error) { - s := &Script{ - v: v, - job: job, - state: NewState(job), - } - if err := s.Validate(); err != nil { - return nil, s.err(err, "invalid script") - } - return s, nil +func (s Script) Validate() error { + return s.Value().Validate("#Script") } -type Action func(context.Context, cue.Value, Fillable) error - -func (s *Script) Run(ctx context.Context, out Fillable) error { - op, err := s.Cue().List() - if err != nil { - return s.err(err, "run") - } - i := 0 - for op.Next() { - // If op is not concrete, interrupt execution without error. - // This allows gradual resolution: compute what you can compute.. leave the rest incomplete. - if !cueIsConcrete(op.Value()) { - debugf("%s: non-concrete op. Leaving script unfinished", op.Value().Path().String()) - return nil - } - if err := s.Do(ctx, op.Value(), out); err != nil { - return s.err(err, "run op %d", i+1) - } - i += 1 - } - return nil -} - -func (s *Script) Do(ctx context.Context, op cue.Value, out Fillable) error { - // Skip no-ops without error (allows more flexible use of if()) - // FIXME: maybe not needed once a clear pattern is established for - // how to use if() in a script? - if cueIsEmptyStruct(op) { - return nil - } - actions := map[string]Action{ - // "#Copy": s.copy, - "#Exec": s.exec, - "#Export": s.export, - "#FetchContainer": s.fetchContainer, - "#FetchGit": s.fetchGit, - "#Load": s.load, - "#Copy": s.copy, - } - for def, action := range actions { - if s.matchSpec(op, def) { - debugf("OP MATCH: %s: %s: %v", def, op.Path().String(), op) - return action(ctx, op, out) - } - } - return fmt.Errorf("[%s] invalid operation: %s", s.v.Path().String(), cueToJSON(op)) -} - -func (s *Script) copy(ctx context.Context, v cue.Value, out Fillable) error { - // Decode copy options - var op struct { - Src string - Dest string - } - if err := v.Decode(&op); err != nil { - return err - } - from := v.Lookup("from") - if isComponent, err := s.job.r.isComponent(from); err != nil { - return err - } else if isComponent { - return s.copyComponent(ctx, from, op.Src, op.Dest) - } - if s.matchSpec(from, "#Script") { - return s.copyScript(ctx, from, op.Src, op.Dest) - } - return fmt.Errorf("copy: invalid source") -} - -func (s *Script) copyScript(ctx context.Context, from cue.Value, src, dest string) error { - // Load source script - fromScript, err := s.job.newScript(from) - if err != nil { - return err - } - // Execute source script - if err := fromScript.Run(ctx, Discard()); err != nil { - return err - } - return s.State().Change(ctx, func(st llb.State) llb.State { - return st.File(llb.Copy( - fromScript.State().LLB(), - src, - dest, - // FIXME: allow more configurable llb options - // For now we define the following convenience presets: - &llb.CopyInfo{ - CopyDirContentsOnly: true, - CreateDestPath: true, - AllowWildcard: true, - }, - )) - }) -} - -func (s *Script) copyComponent(ctx context.Context, from cue.Value, src, dest string) error { - return s.copyScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute")), src, dest) -} - -func (s *Script) load(ctx context.Context, op cue.Value, out Fillable) error { - from := op.Lookup("from") - isComponent, err := s.job.r.isComponent(from) - if err != nil { - return err - } - if isComponent { - debugf("LOAD: from is a component") - return s.loadScript(ctx, from.LookupPath(cue.ParsePath("#dagger.compute"))) - } - if s.matchSpec(from, "#Script") { - return s.loadScript(ctx, from) - } - return fmt.Errorf("load: invalid source") -} - -func (s *Script) loadScript(ctx context.Context, v cue.Value) error { - from, err := s.job.newScript(v) - if err != nil { - return errors.Wrap(err, "load") - } - // NOTE we discard cue outputs from running the loaded script. - // This means we load the LLB state but NOT the cue exports. - // In other words: cue exports are always private to their original location. - if err := from.Run(ctx, Discard()); err != nil { - return errors.Wrap(err, "load/execute") - } - // overwrite buildkit state from loaded from - s.state = from.state - return nil -} - -func (s *Script) exec(ctx context.Context, v cue.Value, out Fillable) error { - var opts []llb.RunOption - var cmd struct { - Args []string - Env map[string]string - Dir string - Always bool - } - v.Decode(&cmd) - // marker for status events - opts = append(opts, llb.WithCustomName(v.Path().String())) - // args - opts = append(opts, llb.Args(cmd.Args)) - // dir - dir := cmd.Dir - if dir == "" { - dir = "/" - } - // env - for k, v := range cmd.Env { - opts = append(opts, llb.AddEnv(k, v)) - } - // always? - if cmd.Always { - cacheBuster, err := randomID(8) - if err != nil { - return err - } - opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster)) - } - // mounts - mnt, _ := v.Lookup("mount").Fields() - for mnt.Next() { - opt, err := s.mount(ctx, mnt.Label(), mnt.Value()) - if err != nil { - return err - } - opts = append(opts, opt) - } - // --> Execute - return s.State().Change(ctx, func(st llb.State) llb.State { - return st.Run(opts...).Root() - }) -} - -func (s *Script) mount(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) { - if s.matchSpec(source, "#MountTmp") { - return llb.AddMount(dest, llb.Scratch(), llb.Tmpfs()), nil - } - if s.matchSpec(source, "#MountCache") { - // FIXME: cache mount - return nil, fmt.Errorf("FIXME: cache mount not yet implemented") - } - if s.matchSpec(source, "#MountScript") { - return s.mountScript(ctx, dest, source) - } - if s.matchSpec(source, "#MountComponent") { - return s.mountComponent(ctx, dest, source) - } - return nil, fmt.Errorf("mount %s to %s: invalid source", source.Path().String(), dest) -} - -// mount when the input is a script (see mountComponent, mountTmpfs, mountCache) -func (s *Script) mountScript(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) { - script, err := s.job.newScript(source) - if err != nil { - return nil, err - } - // FIXME: this is where we re-run everything, - // and rely on solver cache / dedup - if err := script.Run(ctx, Discard()); err != nil { - return nil, err - } - return llb.AddMount(dest, script.State().LLB()), nil -} - -func (s *Script) mountComponent(ctx context.Context, dest string, source cue.Value) (llb.RunOption, error) { - return s.mountScript(ctx, dest, source.LookupPath(cue.ParsePath("from.#dagger.compute"))) -} - -func (s *Script) fetchContainer(ctx context.Context, v cue.Value, out Fillable) error { - var op struct { - Ref string - } - if err := v.Decode(&op); err != nil { - return errors.Wrap(err, "decode fetch-container") - } - return s.State().Change(ctx, llb.Image(op.Ref)) -} - -func (s *Script) fetchGit(ctx context.Context, v cue.Value, out Fillable) error { - // See #FetchGit in spec.cue - var op struct { - Remote string - Ref string - } - if err := v.Decode(&op); err != nil { - return errors.Wrap(err, "decode fetch-git") - } - return s.State().Change(ctx, llb.Git(op.Remote, op.Ref)) -} - -func (s *Script) export(ctx context.Context, v cue.Value, out Fillable) error { - // See #Export in spec.cue - var op struct { - Source string - // FIXME: target - // Target string - Format string - } - v.Decode(&op) - b, err := s.State().ReadFile(ctx, op.Source) - if err != nil { - return err - } - switch op.Format { - case "string": - return out.Fill(string(b)) - case "json": - var o interface{} - if err := json.Unmarshal(b, &o); err != nil { - return err - } - return out.Fill(o) - default: - return fmt.Errorf("unsupported export format: %q", op.Format) - } -} - -func (s *Script) Cue() cue.Value { +func (s *Script) Value() *Value { return s.v } -func (s *Script) Location() string { - return s.Cue().Path().String() -} - -func (s *Script) err(e error, msg string, args ...interface{}) error { - return errors.Wrapf(e, s.Location()+": "+msg, args...) -} - -func (s *Script) Validate() error { - return s.job.r.validateSpec(s.Cue(), "#Script") -} - -func (s *Script) State() *State { - return s.state -} - -func (s *Script) matchSpec(v cue.Value, def string) bool { - // FIXME: we manually filter out empty structs to avoid false positives - // This is necessary because Runtime.ValidateSpec has a bug - // where an empty struct matches everything. - // see https://github.com/cuelang/cue/issues/566#issuecomment-749735878 - // Once the bug is fixed, the manual check can be removed. - if st, err := v.Struct(); err == nil { - if st.Len() == 0 { - debugf("FIXME: manually filtering out empty struct from spec match") - return false +// Run a dagger script +func (s *Script) Execute(ctx context.Context, fs FS, out Fillable) (FS, error) { + err := s.v.RangeList(func(idx int, v *Value) error { + // If op not concrete, interrupt without error. + // This allows gradual resolution: + // compute what you can compute.. leave the rest incomplete. + if !v.IsConcreteR() { + return nil } - } - return s.job.r.matchSpec(v, def) + op, err := v.Op() + if err != nil { + return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len()) + } + fs, err = op.Execute(ctx, fs, out) + if err != nil { + return errors.Wrapf(err, "execute op %d/%d", idx+1, s.v.Len()) + } + return nil + }) + return fs, err +} + +func (s *Script) Walk(fn func(op *Op) error) error { + return s.v.RangeList(func(idx int, v *Value) error { + op, err := v.Op() + if err != nil { + return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len()) + } + if err := op.Walk(fn); err != nil { + return err + } + return nil + }) +} + +func (s *Script) LocalDirs() ([]string, error) { + var dirs []string + err := s.Walk(func(op *Op) error { + if err := op.Validate("#Local"); err != nil { + return nil + } + dir, err := op.Get("dir").String() + if err != nil { + return err + } + dirs = append(dirs, dir) + return nil + }) + return dirs, err } diff --git a/dagger/script_test.go b/dagger/script_test.go new file mode 100644 index 00000000..97b0c021 --- /dev/null +++ b/dagger/script_test.go @@ -0,0 +1,78 @@ +package dagger + +import ( + "strings" + "testing" +) + +func TestWalkBootScript(t *testing.T) { + cc := &Compiler{} + script, err := cc.CompileScript("boot.cue", defaultBootScript) + if err != nil { + t.Fatal(err) + } + dirs, err := script.LocalDirs() + if err != nil { + t.Fatal(err) + } + if len(dirs) != 1 { + t.Fatal(dirs) + } + if dirs[0] != "." { + t.Fatal(dirs) + } +} + +func TestWalkBiggerScript(t *testing.T) { + t.Skip("FIXME") + cc := &Compiler{} + script, err := cc.CompileScript("boot.cue", ` +[ +// { +// do: "load" +// from: { +// do: "local" +// dir: "ga" +// } +// }, + { + do: "local" + dir: "bu" + }, + { + do: "copy" + from: [ + { + do: "local" + dir: "zo" + } + ] + }, + { + do: "exec" + args: ["ls"] + mount: "/mnt": input: [ + { + do: "local" + dir: "meu" + } + ] + } +] +`) + if err != nil { + t.Fatal(err) + } + dirs, err := script.LocalDirs() + if err != nil { + t.Fatal(err) + } + if len(dirs) != 4 { + t.Fatal(dirs) + } + wanted := "ga bu zo meu" + got := strings.Join(dirs, " ") + if wanted != got { + t.Fatal(got) + } +} diff --git a/dagger/solver.go b/dagger/solver.go new file mode 100644 index 00000000..994d2c8a --- /dev/null +++ b/dagger/solver.go @@ -0,0 +1,46 @@ +package dagger + +import ( + "context" + + "github.com/moby/buildkit/client/llb" + bkgw "github.com/moby/buildkit/frontend/gateway/client" +) + +// Polyfill for buildkit gateway client +// Use instead of bkgw.Client +type Solver struct { + c bkgw.Client +} + +func NewSolver(c bkgw.Client) Solver { + return Solver{ + c: c, + } +} + +func (s Solver) FS(input llb.State) FS { + return FS{ + s: s, + input: input, + } +} + +func (s Solver) Scratch() FS { + return s.FS(llb.Scratch()) +} + +func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) { + // marshal llb + def, err := st.Marshal(ctx, llb.LinuxAmd64) + if err != nil { + return nil, err + } + // call solve + res, err := s.c.Solve(ctx, bkgw.SolveRequest{Definition: def.ToPB()}) + if err != nil { + return nil, err + } + // always use single reference (ignore multiple outputs & metadata) + return res.SingleRef() +} diff --git a/dagger/spec.cue b/dagger/spec.cue index f0f8c8c0..7e9098cf 100644 --- a/dagger/spec.cue +++ b/dagger/spec.cue @@ -41,6 +41,7 @@ package dagger // The contents of a #dagger annotation #ComponentConfig: { + // FIXME: deprecated input?: bool // script to compute the value @@ -86,7 +87,7 @@ package dagger #Script: [...#Op] // One operation in a script -#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Load | #Copy +#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy // Export a value from fs state to cue #Export: { @@ -96,22 +97,24 @@ package dagger format: "json"|"yaml"|*"string"|"number"|"boolean" } -#Load: #LoadComponent| #LoadScript -#LoadComponent: { - do: "load" - from: #Component -} -#LoadScript: { - do: "load" - from: #Script +#Local: { + do: "local" + dir: string + include: [...string] | *[] } +// FIXME: bring back load (more efficient than copy) + +#Load: { + do: "load" + from: #Component | #Script +} #Exec: { do: "exec" args: [...string] - env: [string]: string - always: true | *false + env?: [string]: string + always?: true | *false dir: string | *"/" mount?: [string]: #MountTmp | #MountCache | #MountComponent | #MountScript } @@ -141,11 +144,10 @@ package dagger #Copy: { do: "copy" from: #Script | #Component - src: string | *"/" - dest: string | *"/" + src?: string | *"/" + dest?: string | *"/" } - #TestScript: #Script & [ { do: "fetch-container", ref: "alpine:latest" }, { do: "exec", args: ["echo", "hello", "world" ] } diff --git a/dagger/spec.go b/dagger/spec.go new file mode 100644 index 00000000..b5c46e70 --- /dev/null +++ b/dagger/spec.go @@ -0,0 +1,41 @@ +package dagger + +import ( + "fmt" + + "cuelang.org/go/cue" + cueerrors "cuelang.org/go/cue/errors" +) + +// Cue spec validator +type Spec struct { + root *Value +} + +// eg. Validate(op, "#Op") +func (s Spec) Validate(v *Value, defpath string) (err error) { + // Expand cue errors to get full details + // FIXME: there is probably a cleaner way to do this. + defer func() { + if err != nil { + err = fmt.Errorf("%s", cueerrors.Details(err, nil)) + } + }() + + def := s.root.LookupTarget(defpath) + if err := def.Err(); err != nil { + return err + } + if err := def.Unwrap().Fill(v).Validate(cue.Final()); err != nil { + return err + } + return nil +} + +func (s Spec) Match(v *Value, defpath string) bool { + return s.Validate(v, defpath) == nil +} + +func (s Spec) Get(target string) *Value { + return s.root.Get(target) +} diff --git a/dagger/spec_test.go b/dagger/spec_test.go index b1f3b750..597d4025 100644 --- a/dagger/spec_test.go +++ b/dagger/spec_test.go @@ -2,8 +2,6 @@ package dagger import ( "testing" - - "cuelang.org/go/cue" ) func TestMatch(t *testing.T) { @@ -19,39 +17,40 @@ func TestMatch(t *testing.T) { Src: `do: "fetch-git", remote: "github.com/shykes/tests"`, Def: "#FetchGit", }, - { - Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`, - Def: "#Load", - }, - { - Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`, - Def: "#Load", - }, - // Make sure an empty op does NOT match - { - Src: ``, - Def: "", - }, - { - Src: `do: "load" -let package={bash:">3.0"} -from: foo -let foo={#dagger: compute: [ - {do: "fetch-container", ref: "alpine"}, - for pkg, info in package { - if (info & true) != _|_ { - do: "exec" - args: ["echo", "hello", pkg] - } - if (info & string) != _|_ { - do: "exec" - args: ["echo", "hello", pkg, info] - } - } -]} -`, - Def: "#Load", - }, + //FIXME: load is temporarily disabled + // { + // Src: `do: "load", from: [{do: "exec", args: ["echo", "hello"]}]`, + // Def: "#Load", + // }, + // { + // Src: `do: "load", from: #dagger: compute: [{do: "exec", args: ["echo", "hello"]}]`, + // Def: "#Load", + // }, + // // Make sure an empty op does NOT match + // { + // Src: ``, + // Def: "", + // }, + // { + // Src: `do: "load" + //let package={bash:">3.0"} + //from: foo + //let foo={#dagger: compute: [ + // {do: "fetch-container", ref: "alpine"}, + // for pkg, info in package { + // if (info & true) != _|_ { + // do: "exec" + // args: ["echo", "hello", pkg] + // } + // if (info & string) != _|_ { + // do: "exec" + // args: ["echo", "hello", pkg, info] + // } + // } + //]} + //`, + // Def: "#Load", + // }, } for _, d := range data { testMatch(t, d.Src, d.Def) @@ -60,11 +59,11 @@ let foo={#dagger: compute: [ // Test an example op for false positives and negatives func testMatch(t *testing.T, src interface{}, def string) { - r := &Runtime{} - op := compile(t, r, src) + cc := &Compiler{} + op := compile(t, cc, src) if def != "" { - if !r.matchSpec(op, def) { - t.Errorf("false negative: %s: %q", def, src) + if err := op.Validate(def); err != nil { + t.Errorf("false negative: %s: %q: %s", def, src, err) } } for _, cmpDef := range []string{ @@ -72,23 +71,23 @@ func testMatch(t *testing.T, src interface{}, def string) { "#FetchGit", "#FetchContainer", "#Export", - "#Load", "#Copy", + "#Local", } { if cmpDef == def { continue } - if r.matchSpec(op, cmpDef) { + if err := op.Validate(cmpDef); err == nil { t.Errorf("false positive: %s: %q", cmpDef, src) } } return } -func compile(t *testing.T, r *Runtime, src interface{}) cue.Value { - inst, err := r.Compile("", src) +func compile(t *testing.T, cc *Compiler, src interface{}) *Value { + v, err := cc.Compile("", src) if err != nil { t.Fatal(err) } - return inst.Value() + return v } diff --git a/dagger/state.go b/dagger/state.go deleted file mode 100644 index 27be087d..00000000 --- a/dagger/state.go +++ /dev/null @@ -1,53 +0,0 @@ -package dagger - -import ( - "context" - "os" - - "github.com/moby/buildkit/client/llb" - bkgw "github.com/moby/buildkit/frontend/gateway/client" -) - -type State struct { - // Before last solve - input llb.State - // After last solve - output bkgw.Reference - // How to produce the output - s Solver -} - -func NewState(s Solver) *State { - return &State{ - input: llb.Scratch(), - s: s, - } -} - -func (s *State) ReadFile(ctx context.Context, filename string) ([]byte, error) { - if s.output == nil { - return nil, os.ErrNotExist - } - return s.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename}) -} - -func (s *State) Change(ctx context.Context, op interface{}) error { - input := s.input - switch OP := op.(type) { - case llb.State: - input = OP - case func(llb.State) llb.State: - input = OP(input) - } - output, err := s.s.Solve(ctx, input) - if err != nil { - return err - } - s.input = input - s.output = output - return nil -} - -func (s *State) LLB() llb.State { - return s.input -} diff --git a/dagger/types.go b/dagger/types.go new file mode 100644 index 00000000..695e56f1 --- /dev/null +++ b/dagger/types.go @@ -0,0 +1,26 @@ +package dagger + +import ( + "context" +) + +// Implemented by Component, Script, Op +type Executable interface { + Execute(context.Context, FS, Fillable) (FS, error) + Walk(func(*Op) error) error +} + +// Something which can be filled in-place with a cue value +type Fillable interface { + Fill(interface{}) error +} + +func Discard() Fillable { + return discard{} +} + +type discard struct{} + +func (d discard) Fill(x interface{}) error { + return nil +} diff --git a/dagger/utils.go b/dagger/utils.go index 5588372d..0172b1c5 100644 --- a/dagger/utils.go +++ b/dagger/utils.go @@ -4,44 +4,16 @@ import ( "crypto/rand" "encoding/json" "fmt" - "io" - "io/ioutil" "os" - "path/filepath" "strings" "cuelang.org/go/cue" - cueAst "cuelang.org/go/cue/ast" cueerrors "cuelang.org/go/cue/errors" cueformat "cuelang.org/go/cue/format" - cueload "cuelang.org/go/cue/load" - cueParser "cuelang.org/go/cue/parser" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/imagemetaresolver" - "github.com/pkg/errors" ) -// A nil equivalent for cue.Value (when returning errors) -var qnil cue.Value - -type Fillable interface { - Fill(interface{}) error -} - -func Discard() Fillable { - return discard{} -} - -type discard struct{} - -func (d discard) Fill(x interface{}) error { - return nil -} - -type fillableValue struct { - root cue.Value -} - func cuePrint(v cue.Value) (string, error) { b, err := cueformat.Node(v.Syntax()) if err != nil { @@ -50,68 +22,10 @@ func cuePrint(v cue.Value) (string, error) { return string(b), nil } -func (f *fillableValue) Fill(v interface{}) error { - root2 := f.root.Fill(v) - if err := root2.Err(); err != nil { - return err - } - f.root = root2 - return nil -} - -func cueScratch(r *cue.Runtime) Fillable { - f := &fillableValue{} - if inst, err := r.Compile("", ""); err == nil { - f.root = inst.Value() - } - return f -} - func cueErr(err error) error { return fmt.Errorf("%s", cueerrors.Details(err, &cueerrors.Config{})) } -func cueDecodeArray(a cue.Value, idx int, out interface{}) { - a.LookupPath(cue.MakePath(cue.Index(idx))).Decode(out) -} - -func cueToJSON(v cue.Value) JSON { - var out JSON - v.Walk( - func(v cue.Value) bool { - b, err := v.MarshalJSON() - if err == nil { - newOut, err := out.Set(b, cuePathToStrings(v.Path())...) - if err == nil { - out = newOut - } - return false - } - return true - }, - nil, - ) - return out -} - -// Build a cue instance from a directory and args -func cueBuild(r *cue.Runtime, cueRoot string, buildArgs ...string) (*cue.Instance, error) { - var err error - cueRoot, err = filepath.Abs(cueRoot) - if err != nil { - return nil, err - } - buildConfig := &cueload.Config{ - ModuleRoot: cueRoot, - Dir: cueRoot, - } - instances := cueload.Instances(buildArgs, buildConfig) - if len(instances) != 1 { - return nil, errors.New("only one package is supported at a time") - } - return r.Build(instances[0]) -} - func debugJSON(v interface{}) { if os.Getenv("DEBUG") != "" { e := json.NewEncoder(os.Stderr) @@ -144,104 +58,6 @@ func randomID(size int) (string, error) { return fmt.Sprintf("%x", b), nil } -func cueWrapExpr(p string, v cueAst.Expr) (cueAst.Expr, error) { - pExpr, err := cueParser.ParseExpr("path", p) - if err != nil { - return v, err - } - out := v - cursor := pExpr -walk: - for { - switch c := cursor.(type) { - case *cueAst.SelectorExpr: - out = cueAst.NewStruct( - &cueAst.Field{ - Value: out, - Label: c.Sel, - }, - ) - cursor = c.X - case *cueAst.Ident: - out = cueAst.NewStruct( - &cueAst.Field{ - Value: out, - Label: c, - }, - ) - break walk - default: - return out, fmt.Errorf("invalid path expression: %q", p) - } - } - return out, nil -} - -func cueWrapFile(p string, v interface{}) (*cueAst.File, error) { - f, err := cueParser.ParseFile("value", v) - if err != nil { - return f, err - } - decls := make([]cueAst.Decl, 0, len(f.Decls)) - for _, decl := range f.Decls { - switch d := decl.(type) { - case *cueAst.Field: - wrappedExpr, err := cueWrapExpr(p, cueAst.NewStruct(d)) - if err != nil { - return f, err - } - decls = append(decls, &cueAst.EmbedDecl{Expr: wrappedExpr}) - case *cueAst.EmbedDecl: - wrappedExpr, err := cueWrapExpr(p, d.Expr) - if err != nil { - return f, err - } - d.Expr = wrappedExpr - decls = append(decls, d) - case *cueAst.ImportDecl: - decls = append(decls, decl) - default: - fmt.Printf("skipping unsupported decl type %#v\n\n", decl) - continue - } - } - f.Decls = decls - return f, nil -} - -func cueIsEmptyStruct(v cue.Value) bool { - if st, err := v.Struct(); err == nil { - if st.Len() == 0 { - return true - } - } - return false -} - -// Return false if v is not concrete, or contains any -// non-concrete fields or items. -func cueIsConcrete(v cue.Value) bool { - // FIXME: use Value.Walk? - if it, err := v.Fields(); err == nil { - for it.Next() { - if !cueIsConcrete(it.Value()) { - return false - } - } - return true - } - if it, err := v.List(); err == nil { - for it.Next() { - if !cueIsConcrete(it.Value()) { - return false - } - } - return true - } - dv, _ := v.Default() - return v.IsConcrete() || dv.IsConcrete() -} - // LLB Helper to pull a Docker image + all its metadata func llbDockerImage(ref string) llb.State { return llb.Image( @@ -272,17 +88,3 @@ func cueCleanPath(p string) (string, error) { cp := cue.ParsePath(p) return cp.String(), cp.Err() } - -func autoMarshal(value interface{}) ([]byte, error) { - switch v := value.(type) { - case []byte: - return v, nil - case string: - return []byte(v), nil - case io.Reader: - return ioutil.ReadAll(v) - default: - return nil, fmt.Errorf("unsupported marshal inoput type") - } - return []byte(fmt.Sprintf("%v", value)), nil -} diff --git a/dagger/value.go b/dagger/value.go new file mode 100644 index 00000000..daa41541 --- /dev/null +++ b/dagger/value.go @@ -0,0 +1,341 @@ +package dagger + +import ( + "fmt" + "os" + + "cuelang.org/go/cue" + cueformat "cuelang.org/go/cue/format" + "github.com/moby/buildkit/client/llb" +) + +// Polyfill for cue.Value. +// Use instead of cue.Value and cue.Instance +type Value struct { + // FIXME: don't embed, cleaner API + cue.Value + cc *Compiler + inst *cue.Instance +} + +func (v *Value) Lock() { + if v.cc == nil { + return + } + v.cc.Lock() +} + +func (v *Value) Unlock() { + if v.cc == nil { + return + } + v.cc.Unlock() +} + +func (v *Value) Lookup(path ...string) *Value { + v.Lock() + defer v.Unlock() + return v.Wrap(v.Unwrap().Lookup(path...)) +} + +func (v *Value) LookupPath(p cue.Path) *Value { + v.Lock() + defer v.Unlock() + return v.Wrap(v.Unwrap().LookupPath(p)) +} + +// FIXME: deprecated by Get() +func (v *Value) LookupTarget(target string) *Value { + return v.LookupPath(cue.ParsePath(target)) +} + +func (v *Value) Get(target string) *Value { + return v.LookupPath(cue.ParsePath(target)) +} + +// Component returns the component value if v is a valid dagger component or an error otherwise. +// If no '#dagger' annotation is present, os.ErrNotExist +// is returned. +func (v *Value) Component() (*Component, error) { + c := &Component{ + v: v, + } + if !c.Exists() { + return c, os.ErrNotExist + } + if err := c.Validate(); err != nil { + return c, err + } + return c, nil +} + +func (v *Value) Script() (*Script, error) { + s := &Script{ + v: v, + } + if err := s.Validate(); err != nil { + return s, err + } + return s, nil +} + +func (v *Value) Executable() (Executable, error) { + if script, err := v.Script(); err == nil { + return script, nil + } + if component, err := v.Component(); err == nil { + return component, nil + } + if op, err := v.Op(); err == nil { + return op, nil + } + return nil, fmt.Errorf("value is not executable") +} + +// ScriptOrComponent returns one of: +// (1) the component value if v is a valid component (type *Component) +// (2) the script value if v is a valid script (type *Script) +// (3) an error otherwise +func (v *Value) ScriptOrComponent() (interface{}, error) { + s, err := v.Script() + if err == nil { + return s, nil + } + c, err := v.Component() + if err == nil { + return c, nil + } + return nil, fmt.Errorf("not a script or component") +} + +func (v *Value) Op() (*Op, error) { + // Merge #Op definition from spec to get default values + spec, err := v.Compiler().Spec() + if err != nil { + return nil, err + } + v, err = spec.Get("#Op").Merge(v) + if err != nil { + return nil, err + } + op := &Op{ + v: v, + } + return op, nil +} + +func (v *Value) Mount(dest string) (*Mount, error) { + mnt := &Mount{ + v: v, + dest: dest, + } + return mnt, mnt.Validate() +} + +// Interpret this value as a spec +func (v *Value) Spec() (*Spec, error) { + // Spec must be a struct + if _, err := v.Struct(); err != nil { + return nil, err + } + return &Spec{ + root: v, + }, nil +} + +// FIXME: receive string path? +func (v *Value) Merge(x interface{}, path ...string) (*Value, error) { + if xval, ok := x.(*Value); ok { + if xval.Compiler() != v.Compiler() { + return nil, fmt.Errorf("can't merge values from different compilers") + } + x = xval.Unwrap() + } + result := v.Wrap(v.Unwrap().Fill(x, path...)) + return result, result.Validate() +} + +func (v *Value) MergePath(x interface{}, p cue.Path) (*Value, error) { + // FIXME: array indexes and defs are not supported, + // they will be silently converted to regular fields. + // eg. `foo.#bar[0]` will become `foo["#bar"]["0"]` + return v.Merge(x, cuePathToStrings(p)...) +} + +func (v *Value) MergeTarget(x interface{}, target string) (*Value, error) { + return v.MergePath(x, cue.ParsePath(target)) +} + +func (v *Value) RangeList(fn func(int, *Value) error) error { + it, err := v.List() + if err != nil { + return err + } + i := 0 + for it.Next() { + if err := fn(i, v.Wrap(it.Value())); err != nil { + return err + } + i += 1 + } + return nil +} + +func (v *Value) RangeStruct(fn func(string, *Value) error) error { + it, err := v.Fields() + if err != nil { + return err + } + for it.Next() { + if err := fn(it.Label(), v.Wrap(it.Value())); err != nil { + return err + } + } + return nil +} + +// Recursive concreteness check. +// Return false if v is not concrete, or contains any +// non-concrete fields or items. +func (v *Value) IsConcreteR() bool { + // FIXME: use Value.Walk + if it, err := v.Fields(); err == nil { + for it.Next() { + w := v.Wrap(it.Value()) + if !w.IsConcreteR() { + return false + } + } + return true + } + if it, err := v.List(); err == nil { + for it.Next() { + w := v.Wrap(it.Value()) + if !w.IsConcreteR() { + return false + } + } + return true + } + dv, _ := v.Default() + return v.IsConcrete() || dv.IsConcrete() +} + +// Export concrete values to JSON. ignoring non-concrete values. +// Contrast with cue.Value.MarshalJSON which requires all values +// to be concrete. +func (v *Value) JSON() JSON { + v.Lock() + defer v.Unlock() + var out JSON + v.Walk( + func(v cue.Value) bool { + b, err := v.MarshalJSON() + if err == nil { + newOut, err := out.Set(b, cuePathToStrings(v.Path())...) + if err == nil { + out = newOut + } + return false + } + return true + }, + nil, + ) + return out +} + +func (v *Value) SaveJSON(fs FS, filename string) FS { + return fs.Change(func(st llb.State) llb.State { + return st.File( + llb.Mkfile(filename, 0600, v.JSON()), + ) + }) +} + +func (v *Value) Save(fs FS, filename string) (FS, error) { + src, err := v.Source() + if err != nil { + return fs, err + } + return fs.Change(func(st llb.State) llb.State { + return st.File( + llb.Mkfile(filename, 0600, src), + ) + }), nil +} + +func (v *Value) Validate(defs ...string) error { + if err := v.Unwrap().Validate(); err != nil { + return err + } + if len(defs) == 0 { + return nil + } + spec, err := v.Compiler().Spec() + if err != nil { + return err + } + for _, def := range defs { + if err := spec.Validate(v, def); err != nil { + return err + } + } + return nil +} + +// Value implements Fillable. +// This is the only method which changes the value in-place. +// FIXME this co-exists awkwardly with the rest of Value. +func (v *Value) Fill(x interface{}) error { + v.Value = v.Value.Fill(x) + return v.Validate() +} + +func (v *Value) Source() ([]byte, error) { + v.Lock() + defer v.Unlock() + return cueformat.Node(v.Eval().Syntax()) +} + +func (v *Value) IsEmptyStruct() bool { + if st, err := v.Struct(); err == nil { + if st.Len() == 0 { + return true + } + } + return false +} + +func (v *Value) CueInst() *cue.Instance { + return v.inst +} + +func (v *Value) Compiler() *Compiler { + if v.cc == nil { + return &Compiler{} + } + return v.cc +} + +func (v *Value) Debugf(msg string, args ...interface{}) { + prefix := v.Path().String() + args = append([]interface{}{prefix}, args...) + debugf("%s: "+msg, args...) +} + +func (v *Value) Wrap(v2 cue.Value) *Value { + return wrapValue(v2, v.inst, v.cc) +} + +func (v *Value) Unwrap() cue.Value { + return v.Value +} + +func wrapValue(v cue.Value, inst *cue.Instance, cc *Compiler) *Value { + return &Value{ + Value: v, + cc: cc, + inst: inst, + } +} diff --git a/dagger/value_test.go b/dagger/value_test.go new file mode 100644 index 00000000..fdbebae6 --- /dev/null +++ b/dagger/value_test.go @@ -0,0 +1,35 @@ +package dagger + +import ( + "testing" +) + +func TestSimple(t *testing.T) { + cc := &Compiler{} + _, err := cc.EmptyStruct() + if err != nil { + t.Fatal(err) + } +} + +func TestCompileBootScript(t *testing.T) { + cc := &Compiler{} + s, err := cc.CompileScript("boot.cue", defaultBootScript) + if err != nil { + t.Fatal(err) + } + if err := s.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestCompileSimpleScript(t *testing.T) { + cc := &Compiler{} + s, err := cc.CompileScript("simple.cue", `[{do: "local", dir: "."}]`) + if err != nil { + t.Fatal(err) + } + if err := s.Validate(); err != nil { + t.Fatal(err) + } +} diff --git a/main.go b/main.go deleted file mode 100644 index fcb1040b..00000000 --- a/main.go +++ /dev/null @@ -1,43 +0,0 @@ -// A simple main.go for testing the dagger Go API -package main - -import ( - "context" - "fmt" - "os" - - "dagger.cloud/go/dagger" -) - -func main() { - ctx := context.TODO() - c, err := dagger.NewClient(ctx, "") - if err != nil { - fatal(err) - } - - configPath := "." - if len(os.Args) > 1 { - configPath = os.Args[1] - } - - if err := c.SetConfig(configPath); err != nil { - fatal(err) - } - - // if err := c.ConnectInput("source", os.Getenv("HOME")+"/Documents/github/samalba/hello-go"); err != nil { - // fatal(err) - // } - if err := c.Run(ctx, "compute"); err != nil { - fatal(err) - } -} - -func fatalf(msg string, args ...interface{}) { - fmt.Fprintf(os.Stderr, msg, args...) - os.Exit(1) -} - -func fatal(msg interface{}) { - fatalf("%s\n", msg) -}