diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 7cf7bfab..3c22b583 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -13,9 +13,6 @@ import ( ) var ( - // FIXME: global shared cue compiler is a workaround to limitation in the cue API - // This can be made cleaner by moving InputValue (or equivalent) under Env. - cc = &dagger.Compiler{} input *dagger.InputValue updater *dagger.InputValue ) @@ -35,7 +32,7 @@ var computeCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(appcontext.Context()) - env, err := dagger.NewEnv(cc) + env, err := dagger.NewEnv() if err != nil { lg.Fatal().Err(err).Msg("unable to initialize environment") } @@ -68,7 +65,7 @@ var computeCmd = &cobra.Command{ func init() { var err error // Setup --input-* flags - input, err = dagger.NewInputValue(cc, "{}") + input, err = dagger.NewInputValue("{}") if err != nil { panic(err) } @@ -78,7 +75,7 @@ func init() { computeCmd.Flags().Var(input.CueFlag(), "input-cue", "CUE") // Setup (future) --from-* flags - updater, err = dagger.NewInputValue(cc, "[...{do:string, ...}]") + updater, err = dagger.NewInputValue("[...{do:string, ...}]") if err != nil { panic(err) } diff --git a/dagger/build.go b/dagger/build.go new file mode 100644 index 00000000..2bdc8352 --- /dev/null +++ b/dagger/build.go @@ -0,0 +1,59 @@ +package dagger + +import ( + "context" + "path" + "path/filepath" + + cueerrors "cuelang.org/go/cue/errors" + cueload "cuelang.org/go/cue/load" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "dagger.cloud/go/dagger/cc" +) + +// Build a cue configuration tree from the files in fs. +func CueBuild(ctx context.Context, fs FS, args ...string) (*cc.Value, error) { + lg := log.Ctx(ctx) + + // The CUE overlay needs to be prefixed by a non-conflicting path with the + // local filesystem, otherwise Cue will merge the Overlay with whatever Cue + // files it finds locally. + const overlayPrefix = "/config" + + buildConfig := &cueload.Config{ + Dir: overlayPrefix, + Overlay: map[string]cueload.Source{}, + } + buildArgs := args + + err := fs.Walk(ctx, func(p string, f Stat) error { + lg.Debug().Str("path", p).Msg("Compiler.Build: processing") + if f.IsDir() { + return nil + } + if filepath.Ext(p) != ".cue" { + return nil + } + contents, err := fs.ReadFile(ctx, p) + if err != nil { + return errors.Wrap(err, p) + } + overlayPath := path.Join(overlayPrefix, p) + buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents) + return nil + }) + if err != nil { + return nil, err + } + instances := cueload.Instances(buildArgs, buildConfig) + if len(instances) != 1 { + return nil, errors.New("only one package is supported at a time") + } + inst, err := cc.Cue().Build(instances[0]) + if err != nil { + return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{})) + } + return cc.Wrap(inst.Value(), inst), nil +} diff --git a/dagger/cc/cc.go b/dagger/cc/cc.go new file mode 100644 index 00000000..23eaac23 --- /dev/null +++ b/dagger/cc/cc.go @@ -0,0 +1,53 @@ +package cc + +import ( + "cuelang.org/go/cue" + + cueerrors "cuelang.org/go/cue/errors" + "github.com/pkg/errors" +) + +var ( + // Shared global compiler + cc = &Compiler{} +) + +func Compile(name string, src interface{}) (*Value, error) { + return cc.Compile(name, src) +} + +func EmptyStruct() (*Value, error) { + return cc.EmptyStruct() +} + +// FIXME can be refactored away now? +func Wrap(v cue.Value, inst *cue.Instance) *Value { + return cc.Wrap(v, inst) +} + +func Cue() *cue.Runtime { + return cc.Cue() +} + +func Err(err error) error { + if err == nil { + return nil + } + return errors.New(cueerrors.Details(err, &cueerrors.Config{})) +} + +func Lock() { + cc.Lock() +} + +func Unlock() { + cc.Unlock() +} + +func RLock() { + cc.RLock() +} + +func RUnlock() { + cc.RUnlock() +} diff --git a/dagger/cc/compiler.go b/dagger/cc/compiler.go new file mode 100644 index 00000000..19820611 --- /dev/null +++ b/dagger/cc/compiler.go @@ -0,0 +1,40 @@ +package cc + +import ( + "sync" + + "cuelang.org/go/cue" +) + +// Polyfill for a cue runtime +// (we call it compiler to avoid confusion with dagger runtime) +// Use this instead of cue.Runtime +type Compiler struct { + sync.RWMutex + cue.Runtime +} + +func (cc *Compiler) Cue() *cue.Runtime { + return &(cc.Runtime) +} + +// Compile an empty struct +func (cc *Compiler) EmptyStruct() (*Value, error) { + return cc.Compile("", "") +} + +func (cc *Compiler) Compile(name string, src interface{}) (*Value, error) { + cc.Lock() + defer cc.Unlock() + + inst, err := cc.Cue().Compile(name, src) + if err != nil { + // FIXME: cleaner way to unwrap cue error details? + return nil, Err(err) + } + return cc.Wrap(inst.Value(), inst), nil +} + +func (cc *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value { + return wrapValue(v, inst) +} diff --git a/dagger/json.go b/dagger/cc/json.go similarity index 99% rename from dagger/json.go rename to dagger/cc/json.go index c0eaf767..9eef5fef 100644 --- a/dagger/json.go +++ b/dagger/cc/json.go @@ -1,4 +1,4 @@ -package dagger +package cc import ( "fmt" diff --git a/dagger/cc/utils.go b/dagger/cc/utils.go new file mode 100644 index 00000000..bf2d6d29 --- /dev/null +++ b/dagger/cc/utils.go @@ -0,0 +1,22 @@ +package cc + +import ( + "cuelang.org/go/cue" +) + +func cueStringsToCuePath(parts ...string) cue.Path { + selectors := make([]cue.Selector, 0, len(parts)) + for _, part := range parts { + selectors = append(selectors, cue.Str(part)) + } + return cue.MakePath(selectors...) +} + +func cuePathToStrings(p cue.Path) []string { + selectors := p.Selectors() + out := make([]string, len(selectors)) + for i, sel := range selectors { + out[i] = sel.String() + } + return out +} diff --git a/dagger/value.go b/dagger/cc/value.go similarity index 82% rename from dagger/value.go rename to dagger/cc/value.go index c7073524..a5762b6d 100644 --- a/dagger/value.go +++ b/dagger/cc/value.go @@ -1,18 +1,14 @@ -package dagger +package cc import ( - "fmt" - "cuelang.org/go/cue" cueformat "cuelang.org/go/cue/format" - "github.com/moby/buildkit/client/llb" ) // Value is a wrapper around cue.Value. // Use instead of cue.Value and cue.Instance type Value struct { val cue.Value - cc *Compiler inst *cue.Instance } @@ -21,21 +17,20 @@ func (v *Value) CueInst() *cue.Instance { } func (v *Value) Wrap(v2 cue.Value) *Value { - return wrapValue(v2, v.inst, v.cc) + return wrapValue(v2, v.inst) } -func wrapValue(v cue.Value, inst *cue.Instance, cc *Compiler) *Value { +func wrapValue(v cue.Value, inst *cue.Instance) *Value { return &Value{ val: v, - cc: cc, inst: inst, } } // Fill the value in-place, unlike Merge which returns a copy. func (v *Value) Fill(x interface{}) error { - v.cc.Lock() - defer v.cc.Unlock() + cc.Lock() + defer cc.Unlock() // If calling Fill() with a Value, we want to use the underlying // cue.Value to fill. @@ -49,8 +44,8 @@ func (v *Value) Fill(x interface{}) error { // LookupPath is a concurrency safe wrapper around cue.Value.LookupPath func (v *Value) LookupPath(p cue.Path) *Value { - v.cc.RLock() - defer v.cc.RUnlock() + cc.RLock() + defer cc.RUnlock() return v.Wrap(v.val.LookupPath(p)) } @@ -142,9 +137,9 @@ func (v *Value) RangeStruct(fn func(string, *Value) error) error { // 1. Check that the value matches the spec. // 2. Merge the value and the spec, and return the result. func (v *Value) Finalize(spec *Value) (*Value, error) { - v.cc.Lock() + cc.Lock() unified := spec.val.Unify(v.val) - v.cc.Unlock() + cc.Unlock() // FIXME: temporary debug message, remove before merging. // fmt.Printf("Finalize:\n spec=%v\n v=%v\n unified=%v", spec.val, v.val, unified) @@ -155,7 +150,7 @@ func (v *Value) Finalize(spec *Value) (*Value, error) { // fix on top (we access individual fields so have an opportunity // to return an error if they are not there). if err := unified.Validate(cue.Final()); err != nil { - return nil, cueErr(err) + return nil, Err(err) } return v.Merge(spec) } @@ -163,15 +158,12 @@ func (v *Value) Finalize(spec *Value) (*Value, error) { // FIXME: receive string path? func (v *Value) Merge(x interface{}, path ...string) (*Value, error) { if xval, ok := x.(*Value); ok { - if xval.cc != v.cc { - return nil, fmt.Errorf("can't merge values from different compilers") - } x = xval.val } - v.cc.Lock() + cc.Lock() result := v.Wrap(v.val.Fill(x, path...)) - v.cc.Unlock() + cc.Unlock() return result, result.Validate() } @@ -234,48 +226,16 @@ func (v *Value) JSON() JSON { return out } -func (v *Value) SaveJSON(fs FS, filename string) FS { - return fs.Change(func(st llb.State) llb.State { - return st.File( - llb.Mkfile(filename, 0600, v.JSON()), - ) - }) -} - -func (v *Value) Save(fs FS, filename string) (FS, error) { - src, err := v.Source() - if err != nil { - return fs, err - } - return fs.Change(func(st llb.State) llb.State { - return st.File( - llb.Mkfile(filename, 0600, src), - ) - }), nil -} - // Check that a value is valid. Optionally check that it matches // all the specified spec definitions.. -func (v *Value) Validate(defs ...string) error { - if err := v.val.Validate(); err != nil { - return err - } - if len(defs) == 0 { - return nil - } - spec := v.cc.Spec() - for _, def := range defs { - if err := spec.Validate(v, def); err != nil { - return err - } - } - return nil +func (v *Value) Validate() error { + return v.val.Validate() } // Return cue source for this value func (v *Value) Source() ([]byte, error) { - v.cc.RLock() - defer v.cc.RUnlock() + cc.RLock() + defer cc.RUnlock() return cueformat.Node(v.val.Eval().Syntax()) } @@ -294,3 +254,7 @@ func (v *Value) IsEmptyStruct() bool { } return false } + +func (v *Value) Cue() cue.Value { + return v.val +} diff --git a/dagger/client.go b/dagger/client.go index 9c6c802c..1b851a72 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -23,6 +23,8 @@ import ( // docker output "github.com/containerd/console" "github.com/moby/buildkit/util/progress/progressui" + + "dagger.cloud/go/dagger/cc" ) const ( @@ -50,8 +52,8 @@ func NewClient(ctx context.Context, host string) (*Client, error) { }, nil } -// FIXME: return completed *Env, instead of *Value -func (c *Client) Compute(ctx context.Context, env *Env) (*Value, error) { +// FIXME: return completed *Env, instead of *cc.Value +func (c *Client) Compute(ctx context.Context, env *Env) (*cc.Value, error) { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) @@ -75,16 +77,16 @@ func (c *Client) Compute(ctx context.Context, env *Env) (*Value, error) { // Spawn output retriever var ( - out *Value + out *cc.Value err error ) eg.Go(func() error { defer outr.Close() - out, err = c.outputfn(gctx, outr, env.cc) + out, err = c.outputfn(gctx, outr) return err }) - return out, cueErr(eg.Wait()) + return out, cc.Err(eg.Wait()) } func (c *Client) buildfn(ctx context.Context, env *Env, ch chan *bk.SolveStatus, w io.WriteCloser) error { @@ -158,7 +160,7 @@ func (c *Client) buildfn(ctx context.Context, env *Env, ch chan *bk.SolveStatus, } // Read tar export stream from buildkit Build(), and extract cue output -func (c *Client) outputfn(ctx context.Context, r io.Reader, cc *Compiler) (*Value, error) { +func (c *Client) outputfn(ctx context.Context, r io.Reader) (*cc.Value, error) { lg := log.Ctx(ctx) // FIXME: merge this into env output. diff --git a/dagger/compiler.go b/dagger/compiler.go deleted file mode 100644 index ba84590f..00000000 --- a/dagger/compiler.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:generate sh gen.sh -package dagger - -import ( - "context" - "path" - "path/filepath" - "sync" - - "cuelang.org/go/cue" - cueerrors "cuelang.org/go/cue/errors" - cueload "cuelang.org/go/cue/load" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -// Polyfill for a cue runtime -// (we call it compiler to avoid confusion with dagger runtime) -// Use this instead of cue.Runtime -type Compiler struct { - sync.RWMutex - cue.Runtime - spec *Spec -} - -func (cc *Compiler) Cue() *cue.Runtime { - return &(cc.Runtime) -} - -func (cc *Compiler) Spec() *Spec { - if cc.spec != nil { - return cc.spec - } - v, err := cc.Compile("spec.cue", DaggerSpec) - if err != nil { - panic(err) - } - cc.spec, err = newSpec(v) - if err != nil { - panic(err) - } - return cc.spec -} - -// Compile an empty struct -func (cc *Compiler) EmptyStruct() (*Value, error) { - return cc.Compile("", "") -} - -func (cc *Compiler) Compile(name string, src interface{}) (*Value, error) { - cc.Lock() - defer cc.Unlock() - - inst, err := cc.Cue().Compile(name, src) - if err != nil { - // FIXME: cleaner way to unwrap cue error details? - return nil, cueErr(err) - } - return cc.Wrap(inst.Value(), inst), nil -} - -// Compile a cue configuration, and load it as a script. -// If the cue configuration is invalid, or does not match the script spec, -// return an error. -func (cc *Compiler) CompileScript(name string, src interface{}) (*Script, error) { - v, err := cc.Compile(name, src) - if err != nil { - return nil, err - } - return NewScript(v) -} - -// Build a cue configuration tree from the files in fs. -func (cc *Compiler) Build(ctx context.Context, fs FS, args ...string) (*Value, error) { - lg := log.Ctx(ctx) - - // The CUE overlay needs to be prefixed by a non-conflicting path with the - // local filesystem, otherwise Cue will merge the Overlay with whatever Cue - // files it finds locally. - const overlayPrefix = "/config" - - buildConfig := &cueload.Config{ - Dir: overlayPrefix, - Overlay: map[string]cueload.Source{}, - } - buildArgs := args - - err := fs.Walk(ctx, func(p string, f Stat) error { - lg.Debug().Str("path", p).Msg("Compiler.Build: processing") - if f.IsDir() { - return nil - } - if filepath.Ext(p) != ".cue" { - return nil - } - contents, err := fs.ReadFile(ctx, p) - if err != nil { - return errors.Wrap(err, p) - } - overlayPath := path.Join(overlayPrefix, p) - buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents) - return nil - }) - if err != nil { - return nil, err - } - instances := cueload.Instances(buildArgs, buildConfig) - if len(instances) != 1 { - return nil, errors.New("only one package is supported at a time") - } - inst, err := cc.Cue().Build(instances[0]) - if err != nil { - return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{})) - } - return cc.Wrap(inst.Value(), inst), nil -} - -func (cc *Compiler) Wrap(v cue.Value, inst *cue.Instance) *Value { - return wrapValue(v, inst, cc) -} diff --git a/dagger/component.go b/dagger/component.go index a1e4d1de..4644413c 100644 --- a/dagger/component.go +++ b/dagger/component.go @@ -4,16 +4,17 @@ import ( "context" "os" + "dagger.cloud/go/dagger/cc" "github.com/pkg/errors" ) type Component struct { // Source value for the component, without spec merged // eg. `{ string, #dagger: compute: [{do:"fetch-container", ...}]}` - v *Value + v *cc.Value } -func NewComponent(v *Value) (*Component, error) { +func NewComponent(v *cc.Value) (*Component, error) { if !v.Exists() { // Component value does not exist return nil, ErrNotExist @@ -23,7 +24,7 @@ func NewComponent(v *Value) (*Component, error) { return nil, ErrNotExist } // Validate & merge with spec - final, err := v.Finalize(v.cc.Spec().Get("#Component")) + final, err := v.Finalize(spec.Get("#Component")) if err != nil { return nil, errors.Wrap(err, "invalid component") } @@ -32,12 +33,12 @@ func NewComponent(v *Value) (*Component, error) { }, nil } -func (c *Component) Value() *Value { +func (c *Component) Value() *cc.Value { return c.v } // Return the contents of the "#dagger" annotation. -func (c *Component) Config() *Value { +func (c *Component) Config() *cc.Value { return c.Value().Get("#dagger") } @@ -78,7 +79,7 @@ func (c *Component) Compute(ctx context.Context, s Solver, out *Fillable) (FS, e // A component implements the Executable interface by returning its // compute script. -// See Value.Executable(). +// See cc.Value.Executable(). func (c *Component) Execute(ctx context.Context, fs FS, out *Fillable) (FS, error) { script, err := c.ComputeScript() if err != nil { diff --git a/dagger/component_test.go b/dagger/component_test.go index a206d857..e7e3db41 100644 --- a/dagger/component_test.go +++ b/dagger/component_test.go @@ -3,10 +3,11 @@ package dagger import ( "context" "testing" + + "dagger.cloud/go/dagger/cc" ) func TestComponentNotExist(t *testing.T) { - cc := &Compiler{} root, err := cc.Compile("root.cue", ` foo: hello: "world" `) @@ -24,7 +25,6 @@ foo: hello: "world" } func TestLoadEmptyComponent(t *testing.T) { - cc := &Compiler{} root, err := cc.Compile("root.cue", ` foo: #dagger: {} `) @@ -40,7 +40,6 @@ foo: #dagger: {} // Test that default values in spec are applied at the component level // See issue #19 func TestComponentDefaults(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", ` #dagger: compute: [ { @@ -79,7 +78,6 @@ func TestComponentDefaults(t *testing.T) { } func TestValidateEmptyComponent(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", "#dagger: compute: _") if err != nil { t.Fatal(err) @@ -91,7 +89,6 @@ func TestValidateEmptyComponent(t *testing.T) { } func TestValidateSimpleComponent(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", `hello: "world", #dagger: { compute: [{do:"local",dir:"foo"}]}`) if err != nil { t.Fatal(err) diff --git a/dagger/env.go b/dagger/env.go index 24f665e0..63a1636a 100644 --- a/dagger/env.go +++ b/dagger/env.go @@ -8,6 +8,8 @@ import ( cueflow "cuelang.org/go/tools/flow" "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "dagger.cloud/go/dagger/cc" ) type Env struct { @@ -20,19 +22,16 @@ type Env struct { updater *Script // Layer 1: base configuration - base *Value + base *cc.Value // Layer 2: user inputs - input *Value + input *cc.Value // Layer 3: computed values - output *Value + output *cc.Value // All layers merged together: base + input + output - state *Value - - // Use the same cue compiler for everything - cc *Compiler + state *cc.Value } func (env *Env) Updater() *Script { @@ -42,10 +41,10 @@ func (env *Env) Updater() *Script { // Set the updater script for this environment. // u may be: // - A compiled script: *Script -// - A compiled value: *Value +// - A compiled value: *cc.Value // - A cue source: string, []byte, io.Reader func (env *Env) SetUpdater(u interface{}) error { - if v, ok := u.(*Value); ok { + if v, ok := u.(*cc.Value); ok { updater, err := NewScript(v) if err != nil { return errors.Wrap(err, "invalid updater script") @@ -60,7 +59,7 @@ func (env *Env) SetUpdater(u interface{}) error { if u == nil { u = "[]" } - updater, err := env.cc.CompileScript("updater", u) + updater, err := CompileScript("updater", u) if err != nil { return err } @@ -68,16 +67,12 @@ func (env *Env) SetUpdater(u interface{}) error { return nil } -func NewEnv(cc *Compiler) (*Env, error) { - if cc == nil { - cc = &Compiler{} - } +func NewEnv() (*Env, error) { empty, err := cc.EmptyStruct() if err != nil { return nil, err } env := &Env{ - cc: cc, base: empty, input: empty, output: empty, @@ -89,20 +84,16 @@ func NewEnv(cc *Compiler) (*Env, error) { return env, nil } -func (env *Env) Compiler() *Compiler { - return env.cc -} - -func (env *Env) State() *Value { +func (env *Env) State() *cc.Value { return env.state } -func (env *Env) Input() *Value { +func (env *Env) Input() *cc.Value { return env.input } func (env *Env) SetInput(i interface{}) error { - if input, ok := i.(*Value); ok { + if input, ok := i.(*cc.Value); ok { return env.set( env.base, input, @@ -112,7 +103,7 @@ func (env *Env) SetInput(i interface{}) error { if i == nil { i = "{}" } - input, err := env.cc.Compile("input", i) + input, err := cc.Compile("input", i) if err != nil { return err } @@ -132,7 +123,7 @@ func (env *Env) Update(ctx context.Context, s Solver) error { } // load cue files produced by updater // FIXME: BuildAll() to force all files (no required package..) - base, err := env.cc.Build(ctx, src) + base, err := CueBuild(ctx, src) if err != nil { return errors.Wrap(err, "base config") } @@ -143,11 +134,11 @@ func (env *Env) Update(ctx context.Context, s Solver) error { ) } -func (env *Env) Base() *Value { +func (env *Env) Base() *cc.Value { return env.base } -func (env *Env) Output() *Value { +func (env *Env) Output() *cc.Value { return env.output } @@ -195,7 +186,7 @@ func (env *Env) LocalDirs(ctx context.Context) (map[string]string, error) { func (env *Env) Components() []*Component { components := []*Component{} env.State().Walk( - func(v *Value) bool { + func(v *cc.Value) bool { c, err := NewComponent(v) if os.IsNotExist(err) { return true @@ -211,30 +202,30 @@ func (env *Env) Components() []*Component { return components } -// FIXME: this is just a 3-way merge. Add var args to Value.Merge. -func (env *Env) set(base, input, output *Value) (err error) { - // FIXME: make this cleaner in *Value by keeping intermediary instances +// FIXME: this is just a 3-way merge. Add var args to cc.Value.Merge. +func (env *Env) set(base, input, output *cc.Value) (err error) { + // FIXME: make this cleaner in *cc.Value by keeping intermediary instances // FIXME: state.CueInst() must return an instance with the same // contents as state.v, for the purposes of cueflow. - // That is not currently how *Value works, so we prepare the cue + // That is not currently how *cc.Value works, so we prepare the cue // instance manually. - // --> refactor the Value API to do this for us. + // --> refactor the cc.Value API to do this for us. stateInst := env.state.CueInst() - stateInst, err = stateInst.Fill(base.val) + stateInst, err = stateInst.Fill(base.Cue()) if err != nil { return errors.Wrap(err, "merge base & input") } - stateInst, err = stateInst.Fill(input.val) + stateInst, err = stateInst.Fill(input.Cue()) if err != nil { return errors.Wrap(err, "merge base & input") } - stateInst, err = stateInst.Fill(output.val) + stateInst, err = stateInst.Fill(output.Cue()) if err != nil { return errors.Wrap(err, "merge output with base & input") } - state := env.cc.Wrap(stateInst.Value(), stateInst) + state := cc.Wrap(stateInst.Value(), stateInst) // commit env.base = base @@ -248,9 +239,9 @@ func (env *Env) set(base, input, output *Value) (err error) { // (Use with FS.Change) func (env *Env) Export(fs FS) (FS, error) { // FIXME: we serialize as JSON to guarantee a self-contained file. - // Value.Save() leaks imports, so requires a shared cue.mod with + // cc.Value.Save() leaks imports, so requires a shared cue.mod with // client which is undesirable. - // Once Value.Save() resolves non-builtin imports with a tree shake, + // Once cc.Value.Save() resolves non-builtin imports with a tree shake, // we can use it here. // FIXME: Exporting base/input/output separately causes merge errors. @@ -272,7 +263,7 @@ func (env *Env) Export(fs FS) (FS, error) { return fs, err } } - fs = state.SaveJSON(fs, "state.cue") + fs = fs.WriteValueJSON("state.cue", state) return fs, nil } @@ -284,11 +275,11 @@ func (env *Env) Compute(ctx context.Context, s Solver) error { flowInst := env.state.CueInst() lg. Debug(). - Str("value", env.cc.Wrap(flowInst.Value(), flowInst).JSON().String()). + Str("value", cc.Wrap(flowInst.Value(), flowInst).JSON().String()). Msg("walking") // Initialize empty output - output, err := env.cc.EmptyStruct() + output, err := cc.EmptyStruct() if err != nil { return err } @@ -326,7 +317,7 @@ func (env *Env) Compute(ctx context.Context, s Solver) error { } // Cueflow match func flowMatchFn := func(v cue.Value) (cueflow.Runner, error) { - if _, err := NewComponent(env.cc.Wrap(v, flowInst)); err != nil { + if _, err := NewComponent(cc.Wrap(v, flowInst)); err != nil { if os.IsNotExist(err) { // Not a component: skip return nil, nil @@ -340,7 +331,7 @@ func (env *Env) Compute(ctx context.Context, s Solver) error { Logger() ctx := lg.WithContext(ctx) - c, err := NewComponent(env.cc.Wrap(t.Value(), flowInst)) + c, err := NewComponent(cc.Wrap(t.Value(), flowInst)) if err != nil { return err } diff --git a/dagger/env_test.go b/dagger/env_test.go index c09b6426..d77748dc 100644 --- a/dagger/env_test.go +++ b/dagger/env_test.go @@ -3,10 +3,12 @@ package dagger import ( "context" "testing" + + "dagger.cloud/go/dagger/cc" ) func TestSimpleEnvSet(t *testing.T) { - env, err := NewEnv(nil) + env, err := NewEnv() if err != nil { t.Fatal(err) } @@ -23,12 +25,12 @@ func TestSimpleEnvSet(t *testing.T) { } func TestSimpleEnvSetFromInputValue(t *testing.T) { - env, err := NewEnv(nil) + env, err := NewEnv() if err != nil { t.Fatal(err) } - v, err := env.Compiler().Compile("", `hello: "world"`) + v, err := cc.Compile("", `hello: "world"`) if err != nil { t.Fatal(err) } @@ -45,12 +47,12 @@ func TestSimpleEnvSetFromInputValue(t *testing.T) { } func TestEnvInputComponent(t *testing.T) { - env, err := NewEnv(nil) + env, err := NewEnv() if err != nil { t.Fatal(err) } - v, err := env.Compiler().Compile("", `foo: #dagger: compute: [{do:"local",dir:"."}]`) + v, err := cc.Compile("", `foo: #dagger: compute: [{do:"local",dir:"."}]`) if err != nil { t.Fatal(err) } diff --git a/dagger/fs.go b/dagger/fs.go index 1dd365f4..3234deac 100644 --- a/dagger/fs.go +++ b/dagger/fs.go @@ -10,6 +10,8 @@ import ( bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/pkg/errors" fstypes "github.com/tonistiigi/fsutil/types" + + "dagger.cloud/go/dagger/cc" ) type Stat struct { @@ -25,6 +27,26 @@ type FS struct { s Solver } +func (fs FS) WriteValueJSON(filename string, v *cc.Value) FS { + return fs.Change(func(st llb.State) llb.State { + return st.File( + llb.Mkfile(filename, 0600, v.JSON()), + ) + }) +} + +func (fs FS) WriteValueCUE(filename string, v *cc.Value) (FS, error) { + src, err := v.Source() + if err != nil { + return fs, err + } + return fs.Change(func(st llb.State) llb.State { + return st.File( + llb.Mkfile(filename, 0600, src), + ) + }), nil +} + func (fs FS) Solver() Solver { return fs.s } diff --git a/dagger/input.go b/dagger/input.go index b0bf1ba4..42f9ca17 100644 --- a/dagger/input.go +++ b/dagger/input.go @@ -8,16 +8,17 @@ import ( "cuelang.org/go/cue" "github.com/spf13/pflag" + + "dagger.cloud/go/dagger/cc" ) // A mutable cue value with an API suitable for user inputs, // such as command-line flag parsing. type InputValue struct { - root *Value - cc *Compiler + root *cc.Value } -func (iv *InputValue) Value() *Value { +func (iv *InputValue) Value() *cc.Value { return iv.root } @@ -26,21 +27,20 @@ func (iv *InputValue) String() string { return s } -func NewInputValue(cc *Compiler, base interface{}) (*InputValue, error) { +func NewInputValue(base interface{}) (*InputValue, error) { root, err := cc.Compile("base", base) if err != nil { return nil, err } return &InputValue{ - cc: cc, root: root, }, nil } -func (iv *InputValue) Set(s string, enc func(string, *Compiler) (interface{}, error)) error { +func (iv *InputValue) Set(s string, enc func(string) (interface{}, error)) error { // Split from eg. 'foo.bar={bla:"bla"}` k, vRaw := splitkv(s) - v, err := enc(vRaw, iv.cc) + v, err := enc(vRaw) if err != nil { return err } @@ -64,7 +64,7 @@ type stringFlag struct { } func (sf stringFlag) Set(s string) error { - return sf.iv.Set(s, func(s string, _ *Compiler) (interface{}, error) { + return sf.iv.Set(s, func(s string) (interface{}, error) { return s, nil }) } @@ -95,7 +95,7 @@ type dirFlag struct { } func (f dirFlag) Set(s string) error { - return f.iv.Set(s, func(s string, cc *Compiler) (interface{}, error) { + return f.iv.Set(s, func(s string) (interface{}, error) { // FIXME: this is a hack because cue API can't merge into a list include, err := json.Marshal(f.include) if err != nil { @@ -130,7 +130,7 @@ type gitFlag struct { } func (f gitFlag) Set(s string) error { - return f.iv.Set(s, func(s string, cc *Compiler) (interface{}, error) { + return f.iv.Set(s, func(s string) (interface{}, error) { u, err := url.Parse(s) if err != nil { return nil, fmt.Errorf("invalid git url") @@ -170,7 +170,7 @@ type sourceFlag struct { } func (f sourceFlag) Set(s string) error { - return f.iv.Set(s, func(s string, cc *Compiler) (interface{}, error) { + return f.iv.Set(s, func(s string) (interface{}, error) { u, err := url.Parse(s) if err != nil { return nil, err @@ -209,7 +209,7 @@ type cueFlag struct { } func (f cueFlag) Set(s string) error { - return f.iv.Set(s, func(s string, cc *Compiler) (interface{}, error) { + return f.iv.Set(s, func(s string) (interface{}, error) { return cc.Compile("cue input", s) }) } diff --git a/dagger/input_test.go b/dagger/input_test.go index d7172edc..8f919ed5 100644 --- a/dagger/input_test.go +++ b/dagger/input_test.go @@ -6,12 +6,12 @@ import ( ) func TestEnvInputFlag(t *testing.T) { - env, err := NewEnv(nil) + env, err := NewEnv() if err != nil { t.Fatal(err) } - input, err := NewInputValue(env.Compiler(), `{}`) + input, err := NewInputValue(`{}`) if err != nil { t.Fatal(err) } diff --git a/dagger/mount.go b/dagger/mount.go index 790d5ed1..da162047 100644 --- a/dagger/mount.go +++ b/dagger/mount.go @@ -5,14 +5,16 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/pkg/errors" + + "dagger.cloud/go/dagger/cc" ) type Mount struct { dest string - v *Value + v *cc.Value } -func newMount(v *Value, dest string) (*Mount, error) { +func newMount(v *cc.Value, dest string) (*Mount, error) { if !v.Exists() { return nil, ErrNotExist } @@ -22,19 +24,15 @@ func newMount(v *Value, dest string) (*Mount, error) { }, nil } -func (mnt *Mount) Validate(defs ...string) error { - return mnt.v.Validate(append(defs, "#Mount")...) -} - func (mnt *Mount) LLB(ctx context.Context, s Solver) (llb.RunOption, error) { - if err := mnt.Validate("#MountTmp"); err == nil { + if err := spec.Validate(mnt.v, "#MountTmp"); err == nil { return llb.AddMount( mnt.dest, llb.Scratch(), llb.Tmpfs(), ), nil } - if err := mnt.Validate("#MountCache"); err == nil { + if err := spec.Validate(mnt.v, "#MountCache"); err == nil { return llb.AddMount( mnt.dest, llb.Scratch(), diff --git a/dagger/op.go b/dagger/op.go index 9a91da0c..f31d60b0 100644 --- a/dagger/op.go +++ b/dagger/op.go @@ -9,15 +9,16 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" + + "dagger.cloud/go/dagger/cc" ) type Op struct { - v *Value + v *cc.Value } -func NewOp(v *Value) (*Op, error) { - spec := v.cc.Spec().Get("#Op") - final, err := spec.Merge(v) +func NewOp(v *cc.Value) (*Op, error) { + final, err := spec.Get("#Op").Merge(v) if err != nil { return nil, errors.Wrap(err, "invalid op") } @@ -25,7 +26,8 @@ func NewOp(v *Value) (*Op, error) { } // Same as newOp, but without spec merge + validation. -func newOp(v *Value) (*Op, error) { +func newOp(v *cc.Value) (*Op, error) { + // Exists() appears to be buggy, is it needed here? if !v.Exists() { return nil, ErrNotExist } @@ -57,7 +59,7 @@ func (op *Op) Walk(ctx context.Context, fn func(*Op) error) error { // FIXME: we tolerate "from" which is not executable } if err := op.Validate("#Exec"); err == nil { - return op.Get("mount").RangeStruct(func(k string, v *Value) error { + return op.Get("mount").RangeStruct(func(k string, v *cc.Value) error { if from, err := newExecutable(op.Get("from")); err == nil { if err := from.Walk(ctx, fn); err != nil { return err @@ -104,9 +106,11 @@ func (op *Op) Action() (Action, error) { return nil, fmt.Errorf("invalid operation: %s", op.v.JSON()) } -func (op *Op) Validate(defs ...string) error { - defs = append(defs, "#Op") - return op.v.Validate(defs...) +func (op *Op) Validate(def string) error { + if err := spec.Validate(op.v, "#Op"); err != nil { + return err + } + return spec.Validate(op.v, def) } func (op *Op) Subdir(ctx context.Context, fs FS, out *Fillable) (FS, error) { @@ -212,7 +216,7 @@ func (op *Op) Exec(ctx context.Context, fs FS, out *Fillable) (FS, error) { } // mounts if mounts := op.v.Lookup("mount"); mounts.Exists() { - if err := mounts.RangeStruct(func(k string, v *Value) error { + if err := mounts.RangeStruct(func(k string, v *cc.Value) error { mnt, err := newMount(v, k) if err != nil { return err @@ -345,6 +349,6 @@ func (op *Op) FetchGit(ctx context.Context, fs FS, out *Fillable) (FS, error) { return fs.Set(llb.Git(remote, ref)), nil // lazy solve } -func (op *Op) Get(target string) *Value { +func (op *Op) Get(target string) *cc.Value { return op.v.Get(target) } diff --git a/dagger/op_test.go b/dagger/op_test.go index b67cb8c9..3f58c21a 100644 --- a/dagger/op_test.go +++ b/dagger/op_test.go @@ -3,12 +3,13 @@ package dagger import ( "context" "testing" + + "dagger.cloud/go/dagger/cc" ) func TestLocalMatch(t *testing.T) { ctx := context.TODO() - cc := &Compiler{} src := `do: "local", dir: "foo"` v, err := cc.Compile("", src) if err != nil { @@ -34,7 +35,6 @@ func TestLocalMatch(t *testing.T) { func TestCopyMatch(t *testing.T) { ctx := context.TODO() - cc := &Compiler{} src := `do: "copy", from: [{do: "local", dir: "foo"}]` v, err := cc.Compile("", src) if err != nil { diff --git a/dagger/script.go b/dagger/script.go index c744e7ac..a0c3a2ba 100644 --- a/dagger/script.go +++ b/dagger/script.go @@ -6,6 +6,8 @@ import ( "cuelang.org/go/cue" "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "dagger.cloud/go/dagger/cc" ) var ( @@ -13,12 +15,23 @@ var ( ) type Script struct { - v *Value + v *cc.Value } -func NewScript(v *Value) (*Script, error) { +// Compile a cue configuration, and load it as a script. +// If the cue configuration is invalid, or does not match the script spec, +// return an error. +func CompileScript(name string, src interface{}) (*Script, error) { + v, err := cc.Compile(name, src) + if err != nil { + return nil, err + } + return NewScript(v) +} + +func NewScript(v *cc.Value) (*Script, error) { // Validate & merge with spec - final, err := v.Finalize(v.cc.Spec().Get("#Script")) + final, err := v.Finalize(spec.Get("#Script")) if err != nil { return nil, errors.Wrap(err, "invalid script") } @@ -26,7 +39,7 @@ func NewScript(v *Value) (*Script, error) { } // Same as newScript, but without spec merge + validation. -func newScript(v *Value) (*Script, error) { +func newScript(v *cc.Value) (*Script, error) { if !v.Exists() { return nil, ErrNotExist } @@ -35,7 +48,7 @@ func newScript(v *Value) (*Script, error) { }, nil } -func (s *Script) Value() *Value { +func (s *Script) Value() *cc.Value { return s.v } @@ -56,7 +69,7 @@ func (s *Script) Len() uint64 { // Run a dagger script func (s *Script) Execute(ctx context.Context, fs FS, out *Fillable) (FS, error) { - err := s.v.RangeList(func(idx int, v *Value) error { + err := s.v.RangeList(func(idx int, v *cc.Value) error { // If op not concrete, interrupt without error. // This allows gradual resolution: // compute what you can compute.. leave the rest incomplete. @@ -89,7 +102,7 @@ func (s *Script) Execute(ctx context.Context, fs FS, out *Fillable) (FS, error) } func (s *Script) Walk(ctx context.Context, fn func(op *Op) error) error { - return s.v.RangeList(func(idx int, v *Value) error { + return s.v.RangeList(func(idx int, v *cc.Value) error { op, err := newOp(v) if err != nil { return errors.Wrapf(err, "validate op %d/%d", idx+1, s.v.Len()) diff --git a/dagger/script_test.go b/dagger/script_test.go index b6abf3cb..a59a545a 100644 --- a/dagger/script_test.go +++ b/dagger/script_test.go @@ -3,13 +3,14 @@ package dagger import ( "context" "testing" + + "dagger.cloud/go/dagger/cc" ) // Test that a script with missing fields DOES NOT cause an error // NOTE: this behavior may change in the future. func TestScriptMissingFields(t *testing.T) { - cc := &Compiler{} - s, err := cc.CompileScript("test.cue", ` + s, err := CompileScript("test.cue", ` [ { do: "fetch-container" @@ -18,7 +19,7 @@ func TestScriptMissingFields(t *testing.T) { ] `) if err != nil { - t.Fatalf("err=%v\nval=%v\n", err, s.v.val) + t.Fatalf("err=%v\nval=%v\n", err, s.Value().Cue()) } } @@ -78,7 +79,6 @@ func TestScriptLoadComponent(t *testing.T) { // Test that default values in spec are applied func TestScriptDefaults(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", ` [ { @@ -117,12 +117,11 @@ func TestScriptDefaults(t *testing.T) { } func TestValidateEmptyValue(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", "#dagger: compute: _") if err != nil { t.Fatal(err) } - if err := v.Get("#dagger.compute").Validate("#Script"); err != nil { + if err := spec.Validate(v.Get("#dagger.compute"), "#Script"); err != nil { t.Fatal(err) } } @@ -130,7 +129,6 @@ func TestValidateEmptyValue(t *testing.T) { func TestLocalScript(t *testing.T) { ctx := context.TODO() - cc := &Compiler{} src := `[{do: "local", dir: "foo"}]` v, err := cc.Compile("", src) if err != nil { @@ -157,8 +155,7 @@ func TestWalkBiggerScript(t *testing.T) { t.Skip("FIXME") ctx := context.TODO() - cc := &Compiler{} - script, err := cc.CompileScript("boot.cue", ` + script, err := CompileScript("boot.cue", ` [ // { // do: "load" @@ -227,8 +224,7 @@ func TestWalkBiggerScript(t *testing.T) { // Compile a script and check that it has the correct // number of operations. func mkScript(t *testing.T, nOps int, src string) *Script { - cc := &Compiler{} - s, err := cc.CompileScript("test.cue", src) + s, err := CompileScript("test.cue", src) if err != nil { t.Fatal(err) } diff --git a/dagger/spec.go b/dagger/spec.go index bac6a835..ab700b23 100644 --- a/dagger/spec.go +++ b/dagger/spec.go @@ -3,25 +3,35 @@ package dagger import ( cueerrors "cuelang.org/go/cue/errors" "github.com/pkg/errors" + + "dagger.cloud/go/dagger/cc" +) + +var ( + // Global shared dagger spec, generated from spec.cue + spec = NewSpec() ) // Cue spec validator type Spec struct { - root *Value + root *cc.Value } -func newSpec(v *Value) (*Spec, error) { - // Spec contents must be a struct +func NewSpec() *Spec { + v, err := cc.Compile("spec.cue", DaggerSpec) + if err != nil { + panic(err) + } if _, err := v.Struct(); err != nil { - return nil, err + panic(err) } return &Spec{ root: v, - }, nil + } } // eg. Validate(op, "#Op") -func (s Spec) Validate(v *Value, defpath string) error { +func (s Spec) Validate(v *cc.Value, defpath string) error { // Lookup def by name, eg. "#Script" or "#Copy" // See dagger/spec.cue def := s.root.Get(defpath) @@ -32,10 +42,10 @@ func (s Spec) Validate(v *Value, defpath string) error { return nil } -func (s Spec) Match(v *Value, defpath string) bool { +func (s Spec) Match(v *cc.Value, defpath string) bool { return s.Validate(v, defpath) == nil } -func (s Spec) Get(target string) *Value { +func (s Spec) Get(target string) *cc.Value { return s.root.Get(target) } diff --git a/dagger/spec_test.go b/dagger/spec_test.go index eff49ea1..d94e1d5e 100644 --- a/dagger/spec_test.go +++ b/dagger/spec_test.go @@ -2,6 +2,8 @@ package dagger import ( "testing" + + "dagger.cloud/go/dagger/cc" ) func TestMatch(t *testing.T) { @@ -25,10 +27,9 @@ func TestMatch(t *testing.T) { // Test an example op for false positives and negatives func testMatch(t *testing.T, src interface{}, def string) { - cc := &Compiler{} - op := compile(t, cc, src) + op := compile(t, src) if def != "" { - if err := op.Validate(def); err != nil { + if err := spec.Validate(op, def); err != nil { t.Errorf("false negative: %s: %q: %s", def, src, err) } } @@ -43,13 +44,13 @@ func testMatch(t *testing.T, src interface{}, def string) { if cmpDef == def { continue } - if err := op.Validate(cmpDef); err == nil { + if err := spec.Validate(op, cmpDef); err == nil { t.Errorf("false positive: %s: %q", cmpDef, src) } } } -func compile(t *testing.T, cc *Compiler, src interface{}) *Value { +func compile(t *testing.T, src interface{}) *cc.Value { v, err := cc.Compile("", src) if err != nil { t.Fatal(err) diff --git a/dagger/types.go b/dagger/types.go index 70ecbb68..e0340e37 100644 --- a/dagger/types.go +++ b/dagger/types.go @@ -6,6 +6,8 @@ import ( "os" cueflow "cuelang.org/go/tools/flow" + + "dagger.cloud/go/dagger/cc" ) var ErrNotExist = os.ErrNotExist @@ -16,7 +18,7 @@ type Executable interface { Walk(context.Context, func(*Op) error) error } -func newExecutable(v *Value) (Executable, error) { +func newExecutable(v *cc.Value) (Executable, error) { // NOTE: here we need full spec validation, // so we call NewScript, NewComponent, NewOp. if script, err := NewScript(v); err == nil { diff --git a/dagger/utils.go b/dagger/utils.go index cda26e20..211516c8 100644 --- a/dagger/utils.go +++ b/dagger/utils.go @@ -3,19 +3,8 @@ package dagger import ( "crypto/rand" "fmt" - - "cuelang.org/go/cue" - cueerrors "cuelang.org/go/cue/errors" - "github.com/pkg/errors" ) -func cueErr(err error) error { - if err == nil { - return nil - } - return errors.New(cueerrors.Details(err, &cueerrors.Config{})) -} - func randomID(size int) (string, error) { b := make([]byte, size) _, err := rand.Read(b) @@ -24,20 +13,3 @@ func randomID(size int) (string, error) { } return fmt.Sprintf("%x", b), nil } - -func cueStringsToCuePath(parts ...string) cue.Path { - selectors := make([]cue.Selector, 0, len(parts)) - for _, part := range parts { - selectors = append(selectors, cue.Str(part)) - } - return cue.MakePath(selectors...) -} - -func cuePathToStrings(p cue.Path) []string { - selectors := p.Selectors() - out := make([]string, len(selectors)) - for i, sel := range selectors { - out[i] = sel.String() - } - return out -} diff --git a/dagger/value_test.go b/dagger/value_test.go index 2217cb04..52b67986 100644 --- a/dagger/value_test.go +++ b/dagger/value_test.go @@ -2,10 +2,11 @@ package dagger import ( "testing" + + "dagger.cloud/go/dagger/cc" ) func TestValueFinalize(t *testing.T) { - cc := &Compiler{} root, err := cc.Compile("test.cue", ` #FetchContainer: { @@ -59,7 +60,6 @@ func TestValueFinalize(t *testing.T) { // Test that a non-existing field is detected correctly func TestFieldNotExist(t *testing.T) { - cc := &Compiler{} root, err := cc.Compile("test.cue", `foo: "bar"`) if err != nil { t.Fatal(err) @@ -76,7 +76,6 @@ func TestFieldNotExist(t *testing.T) { // Test that a non-existing definition is detected correctly func TestDefNotExist(t *testing.T) { - cc := &Compiler{} root, err := cc.Compile("test.cue", `foo: #bla: "bar"`) if err != nil { t.Fatal(err) @@ -92,7 +91,6 @@ func TestDefNotExist(t *testing.T) { } func TestSimple(t *testing.T) { - cc := &Compiler{} _, err := cc.EmptyStruct() if err != nil { t.Fatal(err) @@ -100,7 +98,6 @@ func TestSimple(t *testing.T) { } func TestJSON(t *testing.T) { - cc := &Compiler{} v, err := cc.Compile("", `foo: hello: "world"`) if err != nil { t.Fatal(err) @@ -117,8 +114,7 @@ func TestJSON(t *testing.T) { } func TestCompileSimpleScript(t *testing.T) { - cc := &Compiler{} - _, err := cc.CompileScript("simple.cue", `[{do: "local", dir: "."}]`) + _, err := CompileScript("simple.cue", `[{do: "local", dir: "."}]`) if err != nil { t.Fatal(err) }