From c4e55a691573b1ef532225d54e9705cce2a28678 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 28 Jan 2021 14:58:13 -0800 Subject: [PATCH] Refactor how client prepares environment. Inputs may now reference local directories Signed-off-by: Solomon Hykes --- cmd/dagger/cmd/compute.go | 12 ++- dagger/client.go | 146 ++++++++-------------------- dagger/component.go | 11 +++ dagger/compute.go | 38 ++++---- dagger/env.go | 200 +++++++++++++++++++++++++++----------- dagger/script.go | 6 +- dagger/script_test.go | 48 ++++----- dagger/value.go | 7 ++ 8 files changed, 251 insertions(+), 217 deletions(-) diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 5fdaf493..87e96af0 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -29,7 +29,7 @@ var computeCmd = &cobra.Command{ c, err := dagger.NewClient(ctx, dagger.ClientConfig{ Input: viper.GetString("input"), - BootDir: args[0], + Updater: localUpdater(args[0]), }) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") @@ -46,6 +46,16 @@ var computeCmd = &cobra.Command{ }, } +func localUpdater(dir string) string { + return fmt.Sprintf(`[ + { + do: "local" + dir: "%s" + include: ["*.cue", "cue.mod"] + } + ]`, dir) +} + func init() { computeCmd.Flags().String("input", "", "Input overlay") diff --git a/dagger/client.go b/dagger/client.go index df1e0db2..8110ff35 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/pkg/errors" @@ -27,25 +28,11 @@ import ( const ( defaultBuildkitHost = "docker-container://buildkitd" - - bkBootKey = "boot" - bkInputKey = "input" - - // Base client config, for default values & schema validation. - baseClientConfig = ` - close({ - bootdir: string | *"." - boot: [...{do:string,...}] | *[ - { - do: "local" - dir: bootdir - include: ["*.cue", "cue.mod"] - } - ] - }) - ` + bkUpdaterKey = "updater" + bkInputKey = "input" ) +// A dagger client type Client struct { c *bk.Client @@ -56,31 +43,54 @@ type Client struct { type ClientConfig struct { // Buildkit host address, eg. `docker://buildkitd` Host string - // Env boot script, eg. `[{do:"local",dir:"."}]` - Boot string - // Env boot dir, eg. `.` - // May be referenced by boot script. - BootDir string - // Input overlay, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]` + // Script to update the env config, eg . `[{do:"local",dir:"."}]` + Updater string + // Input values to merge on the base config, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]` Input string } func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error) { + lg := log.Ctx(ctx) defer func() { if err != nil { // Expand cue errors to get full details err = cueErr(err) } }() - // Finalize config values - localdirs, err := (&cfg).Finalize(ctx) + // Load partial env client-side, to validate & scan local dirs + env, err := NewEnv(cfg.Updater) if err != nil { - return nil, errors.Wrap(err, "client config") + return nil, errors.Wrap(err, "updater") + } + if err := env.SetInput(cfg.Input); err != nil { + return nil, errors.Wrap(err, "input") + } + localdirs, err := env.LocalDirs(ctx) + if err != nil { + return nil, errors.Wrap(err, "scan local dirs") + } + envsrc, err := env.state.SourceString() + if err != nil { + return nil, err + } + lg.Debug(). + Str("func", "NewClient"). + Str("env", envsrc). + Msg("loaded partial env client-side") + for label, dir := range localdirs { + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + localdirs[label] = abs + } + // Configure buildkit client + if cfg.Host == "" { + cfg.Host = os.Getenv("BUILDKIT_HOST") + } + if cfg.Host == "" { + cfg.Host = defaultBuildkitHost } - log.Ctx(ctx).Debug(). - Interface("cfg", cfg). - Interface("localdirs", localdirs). - Msg("finalized client config") c, err := bk.New(ctx, cfg.Host) if err != nil { return nil, errors.Wrap(err, "buildkit client") @@ -92,78 +102,6 @@ func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error }, nil } -// Compile config, fill in final values, -// and return a rollup of local directories -// referenced in the config. -// Localdirs may be referenced in 2 places: -// 1. Boot script -// 2. Input overlay (FIXME: scan not yet implemented) -func (cfg *ClientConfig) Finalize(ctx context.Context) (map[string]string, error) { - localdirs := map[string]string{} - // buildkit client - if cfg.Host == "" { - cfg.Host = os.Getenv("BUILDKIT_HOST") - } - if cfg.Host == "" { - cfg.Host = defaultBuildkitHost - } - // Compile cue template for boot script & boot dir - // (using cue because script may reference dir) - v, err := cfg.Compile() - if err != nil { - return nil, errors.Wrap(err, "invalid client config") - } - // Finalize boot script - boot, err := NewScript(v.Get("boot")) - if err != nil { - return nil, errors.Wrap(err, "invalid env boot script") - } - cfg.Boot = string(boot.Value().JSON()) - // Scan boot script for references to local dirs, to grant access. - bootLocalDirs, err := boot.LocalDirs(ctx) - if err != nil { - return nil, errors.Wrap(err, "scan boot script for local dir access") - } - // Finalize boot dir - cfg.BootDir, err = v.Get("bootdir").String() - if err != nil { - return nil, errors.Wrap(err, "invalid env boot dir") - } - // Scan boot script for references to local dirs, to grant access. - for _, dir := range bootLocalDirs { - // FIXME: randomize local dir references for security - // (currently a malicious cue package may guess common local paths - // and access the corresponding host directory) - localdirs[dir] = dir - } - // FIXME: scan input overlay for references to local dirs, to grant access. - // See issue #41 - return localdirs, nil -} - -// Compile client config to a cue value -// FIXME: include host and input. -func (cfg ClientConfig) Compile() (v *Value, err error) { - cc := &Compiler{} - v, err = cc.Compile("client.cue", baseClientConfig) - if err != nil { - return nil, errors.Wrap(err, "base client config") - } - if cfg.BootDir != "" { - v, err = v.Merge(cfg.BootDir, "bootdir") - if err != nil { - return nil, errors.Wrap(err, "client config key 'bootdir'") - } - } - if cfg.Boot != "" { - v, err = v.Merge(cfg.Boot, "boot") - if err != nil { - return nil, errors.Wrap(err, "client config key 'boot'") - } - } - return v, nil -} - func (c *Client) Compute(ctx context.Context) (*Value, error) { lg := log.Ctx(ctx) @@ -227,8 +165,8 @@ func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.Writ // Setup solve options opts := bk.SolveOpt{ FrontendAttrs: map[string]string{ - bkInputKey: c.cfg.Input, - bkBootKey: c.cfg.Boot, + bkInputKey: c.cfg.Input, + bkUpdaterKey: c.cfg.Updater, }, LocalDirs: c.localdirs, // FIXME: catch output & return as cue value diff --git a/dagger/component.go b/dagger/component.go index 63f1c876..a1e4d1de 100644 --- a/dagger/component.go +++ b/dagger/component.go @@ -46,6 +46,17 @@ func (c *Component) ComputeScript() (*Script, error) { return newScript(c.Config().Get("compute")) } +// Return a list of local dirs required to compute this component. +// (Scanned from the arg `dir` of operations `do: "local"` in the +// compute script. +func (c *Component) LocalDirs(ctx context.Context) (map[string]string, error) { + s, err := c.ComputeScript() + if err != nil { + return nil, err + } + return s.LocalDirs(ctx) +} + // Compute the configuration for this component. // // Difference with Execute: diff --git a/dagger/compute.go b/dagger/compute.go index d615a621..0823d83c 100644 --- a/dagger/compute.go +++ b/dagger/compute.go @@ -13,7 +13,6 @@ import ( // Use by wrapping in a buildkit client Build call, or buildkit frontend. func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) { lg := log.Ctx(ctx) - // FIXME: wrap errors to avoid crashing buildkit Build() // with cue error types (why??) defer func() { @@ -21,37 +20,36 @@ func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) { err = fmt.Errorf("%s", cueerrors.Details(err, nil)) } }() - // Retrieve boot script form client - env, err := NewEnv(ctx, NewSolver(c), getBootScript(c), getInput(c)) + + s := NewSolver(c) + // Retrieve updater script form client + var updater interface{} + if o, exists := c.BuildOpts().Opts[bkUpdaterKey]; exists { + updater = o + } + env, err := NewEnv(updater) if err != nil { return nil, err } + if err := env.Update(ctx, s); err != nil { + return nil, err + } + if input, exists := c.BuildOpts().Opts["input"]; exists { + if err := env.SetInput(input); err != nil { + return nil, err + } + } lg.Debug().Msg("computing env") // Compute output overlay - if err := env.Compute(ctx); err != nil { + if err := env.Compute(ctx, s); err != nil { return nil, err } lg.Debug().Msg("exporting env") // Export env to a cue directory - outdir := NewSolver(c).Scratch() - outdir, err = env.Export(outdir) + outdir, err := env.Export(s.Scratch()) if err != nil { return nil, err } // Wrap cue directory in buildkit result return outdir.Result(ctx) } - -func getBootScript(c bkgw.Client) string { - if boot, exists := c.BuildOpts().Opts["boot"]; exists { - return boot - } - return "" -} - -func getInput(c bkgw.Client) string { - if input, exists := c.BuildOpts().Opts["input"]; exists { - return input - } - return "" -} diff --git a/dagger/env.go b/dagger/env.go index ac4909ac..b8eb6ce0 100644 --- a/dagger/env.go +++ b/dagger/env.go @@ -11,90 +11,139 @@ import ( ) type Env struct { - // Base config + // Env boot script, eg. `[{do:"local",dir:"."}]` + // FIXME: rename to 'update' (script to update the env config) + // FIXME: embed update script in base as '#update' ? + // FIXME: simplify Env by making it single layer? Each layer is one env. + + // Script to update the base configuration + updater *Script + + // Layer 1: base configuration base *Value - // Input overlay: user settings, external directories, secrets... + + // Layer 2: user inputs input *Value - // Output overlay: computed values, generated directories + // Layer 3: computed values output *Value - // Buildkit solver - s Solver - - // Full cue state (base + input + output) + // All layers merged together: base + input + output state *Value - // shared cue compiler - // (because cue API requires shared runtime for everything) + // Use the same cue compiler for everything cc *Compiler } -// Initialize a new environment -func NewEnv(ctx context.Context, s Solver, bootsrc, inputsrc string) (*Env, error) { - lg := log.Ctx(ctx) - - lg. - Debug(). - Str("boot", bootsrc). - Str("input", inputsrc). - Msg("New Env") - - cc := &Compiler{} - // 1. Compile & execute boot script - boot, err := cc.CompileScript("boot.cue", bootsrc) - if err != nil { - return nil, errors.Wrap(err, "compile boot script") +func NewEnv(updater interface{}) (*Env, error) { + var ( + env = &Env{} + cc = &Compiler{} + err error + ) + // 1. Updater + if updater == nil { + updater = "[]" } - bootfs, err := boot.Execute(ctx, s.Scratch(), nil) - if err != nil { - return nil, errors.Wrap(err, "execute boot script") - } - // 2. load cue files produced by boot script - // FIXME: BuildAll() to force all files (no required package..) - lg.Debug().Msg("building cue configuration from boot state") - base, err := cc.Build(ctx, bootfs) - if err != nil { - return nil, errors.Wrap(err, "load base config") - } - // 3. Compile & merge input overlay (user settings, input directories, secrets.) - lg.Debug().Msg("loading input overlay") - input, err := cc.Compile("input.cue", inputsrc) + env.updater, err = cc.CompileScript("updater", updater) if err != nil { return nil, err } - // Merge base + input into a new cue instance - // FIXME: make this cleaner in *Value by keeping intermediary instances - stateInst, err := base.CueInst().Fill(input.CueInst().Value()) + // 2. initialize empty values + empty, err := cc.EmptyStruct() if err != nil { - return nil, errors.Wrap(err, "merge base & input") + return nil, err } - state := cc.Wrap(stateInst.Value(), stateInst) + env.input = empty + env.base = empty + env.state = empty + env.output = empty + // 3. compiler + env.cc = cc + return env, nil +} - lg. - Debug(). - Str("base", base.JSON().String()). - Str("input", input.JSON().String()). - Msg("ENV") +func (env *Env) SetInput(src interface{}) error { + if src == nil { + src = "{}" + } + input, err := env.cc.Compile("input", src) + if err != nil { + return err + } + return env.set( + env.base, + input, + env.output, + ) +} - return &Env{ - base: base, - input: input, - state: state, - s: s, - cc: cc, - }, nil +// Update the base configuration +func (env *Env) Update(ctx context.Context, s Solver) error { + // execute updater script + src, err := env.updater.Execute(ctx, s.Scratch(), nil) + if err != nil { + return err + } + // load cue files produced by updater + // FIXME: BuildAll() to force all files (no required package..) + base, err := env.cc.Build(ctx, src) + if err != nil { + return errors.Wrap(err, "base config") + } + return env.set( + base, + env.input, + env.output, + ) +} + +// Scan all scripts in the environment for references to local directories (do:"local"), +// and return all referenced directory names. +// This is used by clients to grant access to local directories when they are referenced +// by user-specified scripts. +func (env *Env) LocalDirs(ctx context.Context) (map[string]string, error) { + lg := log.Ctx(ctx) + dirs := map[string]string{} + // 1. Walk env state, scan compute script for each component. + lg.Debug().Msg("walking env client-side for local dirs") + _, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error { + lg.Debug(). + Str("func", "Env.LocalDirs"). + Str("component", c.Value().Path().String()). + Msg("scanning next component for local dirs") + cdirs, err := c.LocalDirs(ctx) + if err != nil { + return err + } + for k, v := range cdirs { + dirs[k] = v + } + return nil + }) + if err != nil { + return dirs, err + } + // 2. Scan updater script + updirs, err := env.updater.LocalDirs(ctx) + if err != nil { + return dirs, err + } + for k, v := range updirs { + dirs[k] = v + } + return dirs, nil } // Compute missing values in env configuration, and write them to state. -func (env *Env) Compute(ctx context.Context) error { +func (env *Env) Compute(ctx context.Context, s Solver) error { output, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error { lg := log.Ctx(ctx) lg. Debug(). Msg("[Env.Compute] processing") - if _, err := c.Compute(ctx, env.s, out); err != nil { + if _, err := c.Compute(ctx, s, out); err != nil { lg. Error(). Err(err). @@ -106,7 +155,41 @@ func (env *Env) Compute(ctx context.Context) error { if err != nil { return err } + return env.set( + env.base, + env.input, + output, + ) +} + +// FIXME: this is just a 3-way merge. Add var args to Value.Merge. +func (env *Env) set(base, input, output *Value) error { + // FIXME: make this cleaner in *Value by keeping intermediary instances + // FIXME: state.CueInst() must return an instance with the same + // contents as state.v, for the purposes of cueflow. + // That is not currently how *Value works, so we prepare the cue + // instance manually. + // --> refactor the Value API to do this for us. + baseInst := base.CueInst() + inputInst := input.CueInst() + outputInst := output.CueInst() + + stateInst, err := baseInst.Fill(inputInst.Value()) + if err != nil { + return errors.Wrap(err, "merge base & input") + } + stateInst, err = stateInst.Fill(outputInst.Value()) + if err != nil { + return errors.Wrap(err, "merge output with base & input") + } + + state := env.cc.Wrap(stateInst.Value(), stateInst) + + // commit + env.base = base + env.input = input env.output = output + env.state = state return nil } @@ -135,13 +218,14 @@ func (env *Env) Export(fs FS) (FS, error) { if env.output != nil { state, err = state.Merge(env.output) if err != nil { - return env.s.Scratch(), err + return fs, err } } fs = state.SaveJSON(fs, "state.cue") return fs, nil } +// FIXME: don't need ctx here type EnvWalkFunc func(context.Context, *Component, *Fillable) error // Walk components and return any computed values diff --git a/dagger/script.go b/dagger/script.go index 905da307..93391662 100644 --- a/dagger/script.go +++ b/dagger/script.go @@ -100,8 +100,8 @@ func (s *Script) Walk(ctx context.Context, fn func(op *Op) error) error { }) } -func (s *Script) LocalDirs(ctx context.Context) ([]string, error) { - var dirs []string +func (s *Script) LocalDirs(ctx context.Context) (map[string]string, error) { + dirs := map[string]string{} err := s.Walk(ctx, func(op *Op) error { if err := op.Validate("#Local"); err != nil { // Ignore all operations except 'do:"local"' @@ -111,7 +111,7 @@ func (s *Script) LocalDirs(ctx context.Context) ([]string, error) { if err != nil { return errors.Wrap(err, "invalid 'local' operation") } - dirs = append(dirs, dir) + dirs[dir] = dir return nil }) return dirs, err diff --git a/dagger/script_test.go b/dagger/script_test.go index 4600c553..b6abf3cb 100644 --- a/dagger/script_test.go +++ b/dagger/script_test.go @@ -2,7 +2,6 @@ package dagger import ( "context" - "strings" "testing" ) @@ -154,32 +153,6 @@ func TestLocalScript(t *testing.T) { } } -func TestWalkBootScript(t *testing.T) { - ctx := context.TODO() - - cfg := &ClientConfig{} - _, err := cfg.Finalize(context.TODO()) - if err != nil { - t.Fatal(err) - } - - cc := &Compiler{} - script, err := cc.CompileScript("boot.cue", cfg.Boot) - if err != nil { - t.Fatal(err) - } - dirs, err := script.LocalDirs(ctx) - if err != nil { - t.Fatal(err) - } - if len(dirs) != 1 { - t.Fatal(dirs) - } - if dirs[0] != "." { - t.Fatal(dirs) - } -} - func TestWalkBiggerScript(t *testing.T) { t.Skip("FIXME") @@ -229,10 +202,23 @@ func TestWalkBiggerScript(t *testing.T) { if len(dirs) != 4 { t.Fatal(dirs) } - wanted := "ga bu zo meu" - got := strings.Join(dirs, " ") - if wanted != got { - t.Fatal(got) + wanted := map[string]string{ + "ga": "ga", + "bu": "bu", + "zo": "zo", + "meu": "meu", + } + if len(wanted) != len(dirs) { + t.Fatal(dirs) + } + for k, wantedV := range wanted { + gotV, ok := dirs[k] + if !ok { + t.Fatal(dirs) + } + if gotV != wantedV { + t.Fatal(dirs) + } } } diff --git a/dagger/value.go b/dagger/value.go index ab53e308..e936deb8 100644 --- a/dagger/value.go +++ b/dagger/value.go @@ -249,10 +249,17 @@ func (v *Value) Validate(defs ...string) error { return nil } +// Return cue source for this value func (v *Value) Source() ([]byte, error) { return cueformat.Node(v.val.Eval().Syntax()) } +// Return cue source for this value, as a Go string +func (v *Value) SourceString() (string, error) { + b, err := v.Source() + return string(b), err +} + func (v *Value) IsEmptyStruct() bool { if st, err := v.Struct(); err == nil { if st.Len() == 0 {