Refactor how client prepares environment. Inputs may now reference local directories

Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>

cmd/dagger/cmd/compute.go

@@ -29,7 +29,7 @@ var computeCmd = &cobra.Command{
 
 		c, err := dagger.NewClient(ctx, dagger.ClientConfig{
 			Input:   viper.GetString("input"),
-			BootDir: args[0],
+			Updater: localUpdater(args[0]),
 		})
 		if err != nil {
 			lg.Fatal().Err(err).Msg("unable to create client")
@@ -46,6 +46,16 @@ var computeCmd = &cobra.Command{
 	},
 }
 
+func localUpdater(dir string) string {
+	return fmt.Sprintf(`[
+		{
+			do: "local"
+			dir: "%s"
+			include: ["*.cue", "cue.mod"]
+		}
+	]`, dir)
+}
+
 func init() {
 	computeCmd.Flags().String("input", "", "Input overlay")
 

dagger/client.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -27,25 +28,11 @@ import (
 
 const (
 	defaultBuildkitHost = "docker-container://buildkitd"
-
+	bkUpdaterKey        = "updater"
-	bkBootKey  = "boot"
 	bkInputKey          = "input"
-
-	// Base client config, for default values & schema validation.
-	baseClientConfig = `
-	close({
-		bootdir: string | *"."
-		boot: [...{do:string,...}] | *[
-			{
-				do: "local"
-				dir: bootdir
-				include: ["*.cue", "cue.mod"]
-			}
-		]
-	})
-	`
 )
 
+// A dagger client
 type Client struct {
 	c *bk.Client
 
@@ -56,31 +43,54 @@ type Client struct {
 type ClientConfig struct {
 	// Buildkit host address, eg. `docker://buildkitd`
 	Host string
-	// Env boot script, eg. `[{do:"local",dir:"."}]`
+	// Script to update the env config, eg . `[{do:"local",dir:"."}]`
-	Boot string
+	Updater string
-	// Env boot dir, eg. `.`
+	// Input values to merge on the base config, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]`
-	// May be referenced by boot script.
-	BootDir string
-	// Input overlay, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]`
 	Input string
 }
 
 func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error) {
+	lg := log.Ctx(ctx)
 	defer func() {
 		if err != nil {
 			// Expand cue errors to get full details
 			err = cueErr(err)
 		}
 	}()
-	// Finalize config values
+	// Load partial env client-side, to validate & scan local dirs
-	localdirs, err := (&cfg).Finalize(ctx)
+	env, err := NewEnv(cfg.Updater)
 	if err != nil {
-		return nil, errors.Wrap(err, "client config")
+		return nil, errors.Wrap(err, "updater")
+	}
+	if err := env.SetInput(cfg.Input); err != nil {
+		return nil, errors.Wrap(err, "input")
+	}
+	localdirs, err := env.LocalDirs(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "scan local dirs")
+	}
+	envsrc, err := env.state.SourceString()
+	if err != nil {
+		return nil, err
+	}
+	lg.Debug().
+		Str("func", "NewClient").
+		Str("env", envsrc).
+		Msg("loaded partial env client-side")
+	for label, dir := range localdirs {
+		abs, err := filepath.Abs(dir)
+		if err != nil {
+			return nil, err
+		}
+		localdirs[label] = abs
+	}
+	// Configure buildkit client
+	if cfg.Host == "" {
+		cfg.Host = os.Getenv("BUILDKIT_HOST")
+	}
+	if cfg.Host == "" {
+		cfg.Host = defaultBuildkitHost
 	}
-	log.Ctx(ctx).Debug().
-		Interface("cfg", cfg).
-		Interface("localdirs", localdirs).
-		Msg("finalized client config")
 	c, err := bk.New(ctx, cfg.Host)
 	if err != nil {
 		return nil, errors.Wrap(err, "buildkit client")
@@ -92,78 +102,6 @@ func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error
 	}, nil
 }
 
-// Compile config, fill in final values,
-// and return a rollup of local directories
-// referenced in the config.
-// Localdirs may be referenced in 2 places:
-//  1. Boot script
-//  2. Input overlay (FIXME: scan not yet implemented)
-func (cfg *ClientConfig) Finalize(ctx context.Context) (map[string]string, error) {
-	localdirs := map[string]string{}
-	// buildkit client
-	if cfg.Host == "" {
-		cfg.Host = os.Getenv("BUILDKIT_HOST")
-	}
-	if cfg.Host == "" {
-		cfg.Host = defaultBuildkitHost
-	}
-	// Compile cue template for boot script & boot dir
-	// (using cue because script may reference dir)
-	v, err := cfg.Compile()
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid client config")
-	}
-	// Finalize boot script
-	boot, err := NewScript(v.Get("boot"))
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid env boot script")
-	}
-	cfg.Boot = string(boot.Value().JSON())
-	// Scan boot script for references to local dirs, to grant access.
-	bootLocalDirs, err := boot.LocalDirs(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "scan boot script for local dir access")
-	}
-	// Finalize boot dir
-	cfg.BootDir, err = v.Get("bootdir").String()
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid env boot dir")
-	}
-	// Scan boot script for references to local dirs, to grant access.
-	for _, dir := range bootLocalDirs {
-		// FIXME: randomize local dir references for security
-		// (currently a malicious cue package may guess common local paths
-		//  and access the corresponding host directory)
-		localdirs[dir] = dir
-	}
-	// FIXME: scan input overlay for references to local dirs, to grant access.
-	// See issue #41
-	return localdirs, nil
-}
-
-// Compile client config to a cue value
-// FIXME: include host and input.
-func (cfg ClientConfig) Compile() (v *Value, err error) {
-	cc := &Compiler{}
-	v, err = cc.Compile("client.cue", baseClientConfig)
-	if err != nil {
-		return nil, errors.Wrap(err, "base client config")
-	}
-	if cfg.BootDir != "" {
-		v, err = v.Merge(cfg.BootDir, "bootdir")
-		if err != nil {
-			return nil, errors.Wrap(err, "client config key 'bootdir'")
-		}
-	}
-	if cfg.Boot != "" {
-		v, err = v.Merge(cfg.Boot, "boot")
-		if err != nil {
-			return nil, errors.Wrap(err, "client config key 'boot'")
-		}
-	}
-	return v, nil
-}
-
 func (c *Client) Compute(ctx context.Context) (*Value, error) {
 	lg := log.Ctx(ctx)
 
@@ -228,7 +166,7 @@ func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.Writ
 	opts := bk.SolveOpt{
 		FrontendAttrs: map[string]string{
 			bkInputKey:   c.cfg.Input,
-			bkBootKey:  c.cfg.Boot,
+			bkUpdaterKey: c.cfg.Updater,
 		},
 		LocalDirs: c.localdirs,
 		// FIXME: catch output & return as cue value

dagger/component.go

@@ -46,6 +46,17 @@ func (c *Component) ComputeScript() (*Script, error) {
 	return newScript(c.Config().Get("compute"))
 }
 
+// Return a list of local dirs required to compute this component.
+// (Scanned from the arg `dir` of operations `do: "local"` in the
+// compute script.
+func (c *Component) LocalDirs(ctx context.Context) (map[string]string, error) {
+	s, err := c.ComputeScript()
+	if err != nil {
+		return nil, err
+	}
+	return s.LocalDirs(ctx)
+}
+
 // Compute the configuration for this component.
 //
 // Difference with Execute:

dagger/compute.go

@@ -13,7 +13,6 @@ import (
 // Use by wrapping in a buildkit client Build call, or buildkit frontend.
 func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
 	lg := log.Ctx(ctx)
-
 	// FIXME: wrap errors to avoid crashing buildkit Build()
 	//    with cue error types (why??)
 	defer func() {
@@ -21,37 +20,36 @@ func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
 			err = fmt.Errorf("%s", cueerrors.Details(err, nil))
 		}
 	}()
-	// Retrieve boot script form client
+
-	env, err := NewEnv(ctx, NewSolver(c), getBootScript(c), getInput(c))
+	s := NewSolver(c)
+	// Retrieve updater script form client
+	var updater interface{}
+	if o, exists := c.BuildOpts().Opts[bkUpdaterKey]; exists {
+		updater = o
+	}
+	env, err := NewEnv(updater)
 	if err != nil {
 		return nil, err
 	}
+	if err := env.Update(ctx, s); err != nil {
+		return nil, err
+	}
+	if input, exists := c.BuildOpts().Opts["input"]; exists {
+		if err := env.SetInput(input); err != nil {
+			return nil, err
+		}
+	}
 	lg.Debug().Msg("computing env")
 	// Compute output overlay
-	if err := env.Compute(ctx); err != nil {
+	if err := env.Compute(ctx, s); err != nil {
 		return nil, err
 	}
 	lg.Debug().Msg("exporting env")
 	// Export env to a cue directory
-	outdir := NewSolver(c).Scratch()
+	outdir, err := env.Export(s.Scratch())
-	outdir, err = env.Export(outdir)
 	if err != nil {
 		return nil, err
 	}
 	// Wrap cue directory in buildkit result
 	return outdir.Result(ctx)
 }
-
-func getBootScript(c bkgw.Client) string {
-	if boot, exists := c.BuildOpts().Opts["boot"]; exists {
-		return boot
-	}
-	return ""
-}
-
-func getInput(c bkgw.Client) string {
-	if input, exists := c.BuildOpts().Opts["input"]; exists {
-		return input
-	}
-	return ""
-}

dagger/env.go

@@ -11,90 +11,139 @@ import (
 )
 
 type Env struct {
-	// Base config
+	// Env boot script, eg. `[{do:"local",dir:"."}]`
+	// FIXME: rename to 'update' (script to update the env config)
+	// FIXME: embed update script in base as '#update' ?
+	// FIXME: simplify Env by making it single layer? Each layer is one env.
+
+	// Script to update the base configuration
+	updater *Script
+
+	// Layer 1: base configuration
 	base *Value
-	// Input overlay: user settings, external directories, secrets...
+
+	// Layer 2: user inputs
 	input *Value
 
-	// Output overlay: computed values, generated directories
+	// Layer 3: computed values
 	output *Value
 
-	// Buildkit solver
+	// All layers merged together: base + input + output
-	s Solver
-
-	// Full cue state (base + input + output)
 	state *Value
 
-	// shared cue compiler
+	// Use the same cue compiler for everything
-	// (because cue API requires shared runtime for everything)
 	cc *Compiler
 }
 
-// Initialize a new environment
+func NewEnv(updater interface{}) (*Env, error) {
-func NewEnv(ctx context.Context, s Solver, bootsrc, inputsrc string) (*Env, error) {
+	var (
-	lg := log.Ctx(ctx)
+		env = &Env{}
-
+		cc  = &Compiler{}
-	lg.
+		err error
-		Debug().
+	)
-		Str("boot", bootsrc).
+	// 1. Updater
-		Str("input", inputsrc).
+	if updater == nil {
-		Msg("New Env")
+		updater = "[]"
-
-	cc := &Compiler{}
-	// 1. Compile & execute boot script
-	boot, err := cc.CompileScript("boot.cue", bootsrc)
-	if err != nil {
-		return nil, errors.Wrap(err, "compile boot script")
 	}
-	bootfs, err := boot.Execute(ctx, s.Scratch(), nil)
+	env.updater, err = cc.CompileScript("updater", updater)
-	if err != nil {
-		return nil, errors.Wrap(err, "execute boot script")
-	}
-	// 2. load cue files produced by boot script
-	// FIXME: BuildAll() to force all files (no required package..)
-	lg.Debug().Msg("building cue configuration from boot state")
-	base, err := cc.Build(ctx, bootfs)
-	if err != nil {
-		return nil, errors.Wrap(err, "load base config")
-	}
-	// 3. Compile & merge input overlay (user settings, input directories, secrets.)
-	lg.Debug().Msg("loading input overlay")
-	input, err := cc.Compile("input.cue", inputsrc)
 	if err != nil {
 		return nil, err
 	}
-	// Merge base + input into a new cue instance
+	// 2. initialize empty values
-	// FIXME: make this cleaner in *Value by keeping intermediary instances
+	empty, err := cc.EmptyStruct()
-	stateInst, err := base.CueInst().Fill(input.CueInst().Value())
 	if err != nil {
-		return nil, errors.Wrap(err, "merge base & input")
+		return nil, err
+	}
+	env.input = empty
+	env.base = empty
+	env.state = empty
+	env.output = empty
+	// 3. compiler
+	env.cc = cc
+	return env, nil
 }
-	state := cc.Wrap(stateInst.Value(), stateInst)
 
-	lg.
+func (env *Env) SetInput(src interface{}) error {
-		Debug().
+	if src == nil {
-		Str("base", base.JSON().String()).
+		src = "{}"
-		Str("input", input.JSON().String()).
+	}
-		Msg("ENV")
+	input, err := env.cc.Compile("input", src)
+	if err != nil {
+		return err
+	}
+	return env.set(
+		env.base,
+		input,
+		env.output,
+	)
+}
 
-	return &Env{
+// Update the base configuration
-		base:  base,
+func (env *Env) Update(ctx context.Context, s Solver) error {
-		input: input,
+	// execute updater script
-		state: state,
+	src, err := env.updater.Execute(ctx, s.Scratch(), nil)
-		s:     s,
+	if err != nil {
-		cc:    cc,
+		return err
-	}, nil
+	}
+	// load cue files produced by updater
+	// FIXME: BuildAll() to force all files (no required package..)
+	base, err := env.cc.Build(ctx, src)
+	if err != nil {
+		return errors.Wrap(err, "base config")
+	}
+	return env.set(
+		base,
+		env.input,
+		env.output,
+	)
+}
+
+// Scan all scripts in the environment for references to local directories (do:"local"),
+// and return all referenced directory names.
+// This is used by clients to grant access to local directories when they are referenced
+// by user-specified scripts.
+func (env *Env) LocalDirs(ctx context.Context) (map[string]string, error) {
+	lg := log.Ctx(ctx)
+	dirs := map[string]string{}
+	// 1. Walk env state, scan compute script for each component.
+	lg.Debug().Msg("walking env client-side for local dirs")
+	_, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error {
+		lg.Debug().
+			Str("func", "Env.LocalDirs").
+			Str("component", c.Value().Path().String()).
+			Msg("scanning next component for local dirs")
+		cdirs, err := c.LocalDirs(ctx)
+		if err != nil {
+			return err
+		}
+		for k, v := range cdirs {
+			dirs[k] = v
+		}
+		return nil
+	})
+	if err != nil {
+		return dirs, err
+	}
+	// 2. Scan updater script
+	updirs, err := env.updater.LocalDirs(ctx)
+	if err != nil {
+		return dirs, err
+	}
+	for k, v := range updirs {
+		dirs[k] = v
+	}
+	return dirs, nil
 }
 
 // Compute missing values in env configuration, and write them to state.
-func (env *Env) Compute(ctx context.Context) error {
+func (env *Env) Compute(ctx context.Context, s Solver) error {
 	output, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error {
 		lg := log.Ctx(ctx)
 
 		lg.
 			Debug().
 			Msg("[Env.Compute] processing")
-		if _, err := c.Compute(ctx, env.s, out); err != nil {
+		if _, err := c.Compute(ctx, s, out); err != nil {
 			lg.
 				Error().
 				Err(err).
@@ -106,7 +155,41 @@ func (env *Env) Compute(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	return env.set(
+		env.base,
+		env.input,
+		output,
+	)
+}
+
+// FIXME: this is just a 3-way merge. Add var args to Value.Merge.
+func (env *Env) set(base, input, output *Value) error {
+	// FIXME: make this cleaner in *Value by keeping intermediary instances
+	// FIXME: state.CueInst() must return an instance with the same
+	//  contents as state.v, for the purposes of cueflow.
+	//  That is not currently how *Value works, so we prepare the cue
+	//  instance manually.
+	//   --> refactor the Value API to do this for us.
+	baseInst := base.CueInst()
+	inputInst := input.CueInst()
+	outputInst := output.CueInst()
+
+	stateInst, err := baseInst.Fill(inputInst.Value())
+	if err != nil {
+		return errors.Wrap(err, "merge base & input")
+	}
+	stateInst, err = stateInst.Fill(outputInst.Value())
+	if err != nil {
+		return errors.Wrap(err, "merge output with base & input")
+	}
+
+	state := env.cc.Wrap(stateInst.Value(), stateInst)
+
+	// commit
+	env.base = base
+	env.input = input
 	env.output = output
+	env.state = state
 	return nil
 }
 
@@ -135,13 +218,14 @@ func (env *Env) Export(fs FS) (FS, error) {
 	if env.output != nil {
 		state, err = state.Merge(env.output)
 		if err != nil {
-			return env.s.Scratch(), err
+			return fs, err
 		}
 	}
 	fs = state.SaveJSON(fs, "state.cue")
 	return fs, nil
 }
 
+// FIXME: don't need ctx here
 type EnvWalkFunc func(context.Context, *Component, *Fillable) error
 
 // Walk components and return any computed values

dagger/script.go

@@ -100,8 +100,8 @@ func (s *Script) Walk(ctx context.Context, fn func(op *Op) error) error {
 	})
 }
 
-func (s *Script) LocalDirs(ctx context.Context) ([]string, error) {
+func (s *Script) LocalDirs(ctx context.Context) (map[string]string, error) {
-	var dirs []string
+	dirs := map[string]string{}
 	err := s.Walk(ctx, func(op *Op) error {
 		if err := op.Validate("#Local"); err != nil {
 			// Ignore all operations except 'do:"local"'
@@ -111,7 +111,7 @@ func (s *Script) LocalDirs(ctx context.Context) ([]string, error) {
 		if err != nil {
 			return errors.Wrap(err, "invalid 'local' operation")
 		}
-		dirs = append(dirs, dir)
+		dirs[dir] = dir
 		return nil
 	})
 	return dirs, err

dagger/script_test.go

@@ -2,7 +2,6 @@ package dagger
 
 import (
 	"context"
-	"strings"
 	"testing"
 )
 
@@ -154,32 +153,6 @@ func TestLocalScript(t *testing.T) {
 	}
 }
 
-func TestWalkBootScript(t *testing.T) {
-	ctx := context.TODO()
-
-	cfg := &ClientConfig{}
-	_, err := cfg.Finalize(context.TODO())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	cc := &Compiler{}
-	script, err := cc.CompileScript("boot.cue", cfg.Boot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dirs, err := script.LocalDirs(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(dirs) != 1 {
-		t.Fatal(dirs)
-	}
-	if dirs[0] != "." {
-		t.Fatal(dirs)
-	}
-}
-
 func TestWalkBiggerScript(t *testing.T) {
 	t.Skip("FIXME")
 
@@ -229,10 +202,23 @@ func TestWalkBiggerScript(t *testing.T) {
 	if len(dirs) != 4 {
 		t.Fatal(dirs)
 	}
-	wanted := "ga bu zo meu"
+	wanted := map[string]string{
-	got := strings.Join(dirs, " ")
+		"ga":  "ga",
-	if wanted != got {
+		"bu":  "bu",
-		t.Fatal(got)
+		"zo":  "zo",
+		"meu": "meu",
+	}
+	if len(wanted) != len(dirs) {
+		t.Fatal(dirs)
+	}
+	for k, wantedV := range wanted {
+		gotV, ok := dirs[k]
+		if !ok {
+			t.Fatal(dirs)
+		}
+		if gotV != wantedV {
+			t.Fatal(dirs)
+		}
 	}
 }
 

dagger/value.go

@@ -249,10 +249,17 @@ func (v *Value) Validate(defs ...string) error {
 	return nil
 }
 
+// Return cue source for this value
 func (v *Value) Source() ([]byte, error) {
 	return cueformat.Node(v.val.Eval().Syntax())
 }
 
+// Return cue source for this value, as a Go string
+func (v *Value) SourceString() (string, error) {
+	b, err := v.Source()
+	return string(b), err
+}
+
 func (v *Value) IsEmptyStruct() bool {
 	if st, err := v.Struct(); err == nil {
 		if st.Len() == 0 {