Refactor how client prepares environment. Inputs may now reference local directories

Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes 2021-01-28 14:58:13 -08:00
parent 6f4577d501
commit c4e55a6915
8 changed files with 251 additions and 217 deletions

View File

@ -29,7 +29,7 @@ var computeCmd = &cobra.Command{
c, err := dagger.NewClient(ctx, dagger.ClientConfig{
Input: viper.GetString("input"),
BootDir: args[0],
Updater: localUpdater(args[0]),
})
if err != nil {
lg.Fatal().Err(err).Msg("unable to create client")
@ -46,6 +46,16 @@ var computeCmd = &cobra.Command{
},
}
func localUpdater(dir string) string {
return fmt.Sprintf(`[
{
do: "local"
dir: "%s"
include: ["*.cue", "cue.mod"]
}
]`, dir)
}
func init() {
computeCmd.Flags().String("input", "", "Input overlay")

View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
@ -27,25 +28,11 @@ import (
const (
defaultBuildkitHost = "docker-container://buildkitd"
bkBootKey = "boot"
bkInputKey = "input"
// Base client config, for default values & schema validation.
baseClientConfig = `
close({
bootdir: string | *"."
boot: [...{do:string,...}] | *[
{
do: "local"
dir: bootdir
include: ["*.cue", "cue.mod"]
}
]
})
`
bkUpdaterKey = "updater"
bkInputKey = "input"
)
// A dagger client
type Client struct {
c *bk.Client
@ -56,31 +43,54 @@ type Client struct {
type ClientConfig struct {
// Buildkit host address, eg. `docker://buildkitd`
Host string
// Env boot script, eg. `[{do:"local",dir:"."}]`
Boot string
// Env boot dir, eg. `.`
// May be referenced by boot script.
BootDir string
// Input overlay, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]`
// Script to update the env config, eg . `[{do:"local",dir:"."}]`
Updater string
// Input values to merge on the base config, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]`
Input string
}
func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error) {
lg := log.Ctx(ctx)
defer func() {
if err != nil {
// Expand cue errors to get full details
err = cueErr(err)
}
}()
// Finalize config values
localdirs, err := (&cfg).Finalize(ctx)
// Load partial env client-side, to validate & scan local dirs
env, err := NewEnv(cfg.Updater)
if err != nil {
return nil, errors.Wrap(err, "client config")
return nil, errors.Wrap(err, "updater")
}
if err := env.SetInput(cfg.Input); err != nil {
return nil, errors.Wrap(err, "input")
}
localdirs, err := env.LocalDirs(ctx)
if err != nil {
return nil, errors.Wrap(err, "scan local dirs")
}
envsrc, err := env.state.SourceString()
if err != nil {
return nil, err
}
lg.Debug().
Str("func", "NewClient").
Str("env", envsrc).
Msg("loaded partial env client-side")
for label, dir := range localdirs {
abs, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
localdirs[label] = abs
}
// Configure buildkit client
if cfg.Host == "" {
cfg.Host = os.Getenv("BUILDKIT_HOST")
}
if cfg.Host == "" {
cfg.Host = defaultBuildkitHost
}
log.Ctx(ctx).Debug().
Interface("cfg", cfg).
Interface("localdirs", localdirs).
Msg("finalized client config")
c, err := bk.New(ctx, cfg.Host)
if err != nil {
return nil, errors.Wrap(err, "buildkit client")
@ -92,78 +102,6 @@ func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error
}, nil
}
// Compile config, fill in final values,
// and return a rollup of local directories
// referenced in the config.
// Localdirs may be referenced in 2 places:
// 1. Boot script
// 2. Input overlay (FIXME: scan not yet implemented)
func (cfg *ClientConfig) Finalize(ctx context.Context) (map[string]string, error) {
localdirs := map[string]string{}
// buildkit client
if cfg.Host == "" {
cfg.Host = os.Getenv("BUILDKIT_HOST")
}
if cfg.Host == "" {
cfg.Host = defaultBuildkitHost
}
// Compile cue template for boot script & boot dir
// (using cue because script may reference dir)
v, err := cfg.Compile()
if err != nil {
return nil, errors.Wrap(err, "invalid client config")
}
// Finalize boot script
boot, err := NewScript(v.Get("boot"))
if err != nil {
return nil, errors.Wrap(err, "invalid env boot script")
}
cfg.Boot = string(boot.Value().JSON())
// Scan boot script for references to local dirs, to grant access.
bootLocalDirs, err := boot.LocalDirs(ctx)
if err != nil {
return nil, errors.Wrap(err, "scan boot script for local dir access")
}
// Finalize boot dir
cfg.BootDir, err = v.Get("bootdir").String()
if err != nil {
return nil, errors.Wrap(err, "invalid env boot dir")
}
// Scan boot script for references to local dirs, to grant access.
for _, dir := range bootLocalDirs {
// FIXME: randomize local dir references for security
// (currently a malicious cue package may guess common local paths
// and access the corresponding host directory)
localdirs[dir] = dir
}
// FIXME: scan input overlay for references to local dirs, to grant access.
// See issue #41
return localdirs, nil
}
// Compile client config to a cue value
// FIXME: include host and input.
func (cfg ClientConfig) Compile() (v *Value, err error) {
cc := &Compiler{}
v, err = cc.Compile("client.cue", baseClientConfig)
if err != nil {
return nil, errors.Wrap(err, "base client config")
}
if cfg.BootDir != "" {
v, err = v.Merge(cfg.BootDir, "bootdir")
if err != nil {
return nil, errors.Wrap(err, "client config key 'bootdir'")
}
}
if cfg.Boot != "" {
v, err = v.Merge(cfg.Boot, "boot")
if err != nil {
return nil, errors.Wrap(err, "client config key 'boot'")
}
}
return v, nil
}
func (c *Client) Compute(ctx context.Context) (*Value, error) {
lg := log.Ctx(ctx)
@ -227,8 +165,8 @@ func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.Writ
// Setup solve options
opts := bk.SolveOpt{
FrontendAttrs: map[string]string{
bkInputKey: c.cfg.Input,
bkBootKey: c.cfg.Boot,
bkInputKey: c.cfg.Input,
bkUpdaterKey: c.cfg.Updater,
},
LocalDirs: c.localdirs,
// FIXME: catch output & return as cue value

View File

@ -46,6 +46,17 @@ func (c *Component) ComputeScript() (*Script, error) {
return newScript(c.Config().Get("compute"))
}
// Return a list of local dirs required to compute this component.
// (Scanned from the arg `dir` of operations `do: "local"` in the
// compute script.
func (c *Component) LocalDirs(ctx context.Context) (map[string]string, error) {
s, err := c.ComputeScript()
if err != nil {
return nil, err
}
return s.LocalDirs(ctx)
}
// Compute the configuration for this component.
//
// Difference with Execute:

View File

@ -13,7 +13,6 @@ import (
// Use by wrapping in a buildkit client Build call, or buildkit frontend.
func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
lg := log.Ctx(ctx)
// FIXME: wrap errors to avoid crashing buildkit Build()
// with cue error types (why??)
defer func() {
@ -21,37 +20,36 @@ func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
err = fmt.Errorf("%s", cueerrors.Details(err, nil))
}
}()
// Retrieve boot script form client
env, err := NewEnv(ctx, NewSolver(c), getBootScript(c), getInput(c))
s := NewSolver(c)
// Retrieve updater script form client
var updater interface{}
if o, exists := c.BuildOpts().Opts[bkUpdaterKey]; exists {
updater = o
}
env, err := NewEnv(updater)
if err != nil {
return nil, err
}
if err := env.Update(ctx, s); err != nil {
return nil, err
}
if input, exists := c.BuildOpts().Opts["input"]; exists {
if err := env.SetInput(input); err != nil {
return nil, err
}
}
lg.Debug().Msg("computing env")
// Compute output overlay
if err := env.Compute(ctx); err != nil {
if err := env.Compute(ctx, s); err != nil {
return nil, err
}
lg.Debug().Msg("exporting env")
// Export env to a cue directory
outdir := NewSolver(c).Scratch()
outdir, err = env.Export(outdir)
outdir, err := env.Export(s.Scratch())
if err != nil {
return nil, err
}
// Wrap cue directory in buildkit result
return outdir.Result(ctx)
}
func getBootScript(c bkgw.Client) string {
if boot, exists := c.BuildOpts().Opts["boot"]; exists {
return boot
}
return ""
}
func getInput(c bkgw.Client) string {
if input, exists := c.BuildOpts().Opts["input"]; exists {
return input
}
return ""
}

View File

@ -11,90 +11,139 @@ import (
)
type Env struct {
// Base config
// Env boot script, eg. `[{do:"local",dir:"."}]`
// FIXME: rename to 'update' (script to update the env config)
// FIXME: embed update script in base as '#update' ?
// FIXME: simplify Env by making it single layer? Each layer is one env.
// Script to update the base configuration
updater *Script
// Layer 1: base configuration
base *Value
// Input overlay: user settings, external directories, secrets...
// Layer 2: user inputs
input *Value
// Output overlay: computed values, generated directories
// Layer 3: computed values
output *Value
// Buildkit solver
s Solver
// Full cue state (base + input + output)
// All layers merged together: base + input + output
state *Value
// shared cue compiler
// (because cue API requires shared runtime for everything)
// Use the same cue compiler for everything
cc *Compiler
}
// Initialize a new environment
func NewEnv(ctx context.Context, s Solver, bootsrc, inputsrc string) (*Env, error) {
lg := log.Ctx(ctx)
lg.
Debug().
Str("boot", bootsrc).
Str("input", inputsrc).
Msg("New Env")
cc := &Compiler{}
// 1. Compile & execute boot script
boot, err := cc.CompileScript("boot.cue", bootsrc)
if err != nil {
return nil, errors.Wrap(err, "compile boot script")
func NewEnv(updater interface{}) (*Env, error) {
var (
env = &Env{}
cc = &Compiler{}
err error
)
// 1. Updater
if updater == nil {
updater = "[]"
}
bootfs, err := boot.Execute(ctx, s.Scratch(), nil)
if err != nil {
return nil, errors.Wrap(err, "execute boot script")
}
// 2. load cue files produced by boot script
// FIXME: BuildAll() to force all files (no required package..)
lg.Debug().Msg("building cue configuration from boot state")
base, err := cc.Build(ctx, bootfs)
if err != nil {
return nil, errors.Wrap(err, "load base config")
}
// 3. Compile & merge input overlay (user settings, input directories, secrets.)
lg.Debug().Msg("loading input overlay")
input, err := cc.Compile("input.cue", inputsrc)
env.updater, err = cc.CompileScript("updater", updater)
if err != nil {
return nil, err
}
// Merge base + input into a new cue instance
// FIXME: make this cleaner in *Value by keeping intermediary instances
stateInst, err := base.CueInst().Fill(input.CueInst().Value())
// 2. initialize empty values
empty, err := cc.EmptyStruct()
if err != nil {
return nil, errors.Wrap(err, "merge base & input")
return nil, err
}
state := cc.Wrap(stateInst.Value(), stateInst)
env.input = empty
env.base = empty
env.state = empty
env.output = empty
// 3. compiler
env.cc = cc
return env, nil
}
lg.
Debug().
Str("base", base.JSON().String()).
Str("input", input.JSON().String()).
Msg("ENV")
func (env *Env) SetInput(src interface{}) error {
if src == nil {
src = "{}"
}
input, err := env.cc.Compile("input", src)
if err != nil {
return err
}
return env.set(
env.base,
input,
env.output,
)
}
return &Env{
base: base,
input: input,
state: state,
s: s,
cc: cc,
}, nil
// Update the base configuration
func (env *Env) Update(ctx context.Context, s Solver) error {
// execute updater script
src, err := env.updater.Execute(ctx, s.Scratch(), nil)
if err != nil {
return err
}
// load cue files produced by updater
// FIXME: BuildAll() to force all files (no required package..)
base, err := env.cc.Build(ctx, src)
if err != nil {
return errors.Wrap(err, "base config")
}
return env.set(
base,
env.input,
env.output,
)
}
// Scan all scripts in the environment for references to local directories (do:"local"),
// and return all referenced directory names.
// This is used by clients to grant access to local directories when they are referenced
// by user-specified scripts.
func (env *Env) LocalDirs(ctx context.Context) (map[string]string, error) {
lg := log.Ctx(ctx)
dirs := map[string]string{}
// 1. Walk env state, scan compute script for each component.
lg.Debug().Msg("walking env client-side for local dirs")
_, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error {
lg.Debug().
Str("func", "Env.LocalDirs").
Str("component", c.Value().Path().String()).
Msg("scanning next component for local dirs")
cdirs, err := c.LocalDirs(ctx)
if err != nil {
return err
}
for k, v := range cdirs {
dirs[k] = v
}
return nil
})
if err != nil {
return dirs, err
}
// 2. Scan updater script
updirs, err := env.updater.LocalDirs(ctx)
if err != nil {
return dirs, err
}
for k, v := range updirs {
dirs[k] = v
}
return dirs, nil
}
// Compute missing values in env configuration, and write them to state.
func (env *Env) Compute(ctx context.Context) error {
func (env *Env) Compute(ctx context.Context, s Solver) error {
output, err := env.Walk(ctx, func(ctx context.Context, c *Component, out *Fillable) error {
lg := log.Ctx(ctx)
lg.
Debug().
Msg("[Env.Compute] processing")
if _, err := c.Compute(ctx, env.s, out); err != nil {
if _, err := c.Compute(ctx, s, out); err != nil {
lg.
Error().
Err(err).
@ -106,7 +155,41 @@ func (env *Env) Compute(ctx context.Context) error {
if err != nil {
return err
}
return env.set(
env.base,
env.input,
output,
)
}
// FIXME: this is just a 3-way merge. Add var args to Value.Merge.
func (env *Env) set(base, input, output *Value) error {
// FIXME: make this cleaner in *Value by keeping intermediary instances
// FIXME: state.CueInst() must return an instance with the same
// contents as state.v, for the purposes of cueflow.
// That is not currently how *Value works, so we prepare the cue
// instance manually.
// --> refactor the Value API to do this for us.
baseInst := base.CueInst()
inputInst := input.CueInst()
outputInst := output.CueInst()
stateInst, err := baseInst.Fill(inputInst.Value())
if err != nil {
return errors.Wrap(err, "merge base & input")
}
stateInst, err = stateInst.Fill(outputInst.Value())
if err != nil {
return errors.Wrap(err, "merge output with base & input")
}
state := env.cc.Wrap(stateInst.Value(), stateInst)
// commit
env.base = base
env.input = input
env.output = output
env.state = state
return nil
}
@ -135,13 +218,14 @@ func (env *Env) Export(fs FS) (FS, error) {
if env.output != nil {
state, err = state.Merge(env.output)
if err != nil {
return env.s.Scratch(), err
return fs, err
}
}
fs = state.SaveJSON(fs, "state.cue")
return fs, nil
}
// FIXME: don't need ctx here
type EnvWalkFunc func(context.Context, *Component, *Fillable) error
// Walk components and return any computed values

View File

@ -100,8 +100,8 @@ func (s *Script) Walk(ctx context.Context, fn func(op *Op) error) error {
})
}
func (s *Script) LocalDirs(ctx context.Context) ([]string, error) {
var dirs []string
func (s *Script) LocalDirs(ctx context.Context) (map[string]string, error) {
dirs := map[string]string{}
err := s.Walk(ctx, func(op *Op) error {
if err := op.Validate("#Local"); err != nil {
// Ignore all operations except 'do:"local"'
@ -111,7 +111,7 @@ func (s *Script) LocalDirs(ctx context.Context) ([]string, error) {
if err != nil {
return errors.Wrap(err, "invalid 'local' operation")
}
dirs = append(dirs, dir)
dirs[dir] = dir
return nil
})
return dirs, err

View File

@ -2,7 +2,6 @@ package dagger
import (
"context"
"strings"
"testing"
)
@ -154,32 +153,6 @@ func TestLocalScript(t *testing.T) {
}
}
func TestWalkBootScript(t *testing.T) {
ctx := context.TODO()
cfg := &ClientConfig{}
_, err := cfg.Finalize(context.TODO())
if err != nil {
t.Fatal(err)
}
cc := &Compiler{}
script, err := cc.CompileScript("boot.cue", cfg.Boot)
if err != nil {
t.Fatal(err)
}
dirs, err := script.LocalDirs(ctx)
if err != nil {
t.Fatal(err)
}
if len(dirs) != 1 {
t.Fatal(dirs)
}
if dirs[0] != "." {
t.Fatal(dirs)
}
}
func TestWalkBiggerScript(t *testing.T) {
t.Skip("FIXME")
@ -229,10 +202,23 @@ func TestWalkBiggerScript(t *testing.T) {
if len(dirs) != 4 {
t.Fatal(dirs)
}
wanted := "ga bu zo meu"
got := strings.Join(dirs, " ")
if wanted != got {
t.Fatal(got)
wanted := map[string]string{
"ga": "ga",
"bu": "bu",
"zo": "zo",
"meu": "meu",
}
if len(wanted) != len(dirs) {
t.Fatal(dirs)
}
for k, wantedV := range wanted {
gotV, ok := dirs[k]
if !ok {
t.Fatal(dirs)
}
if gotV != wantedV {
t.Fatal(dirs)
}
}
}

View File

@ -249,10 +249,17 @@ func (v *Value) Validate(defs ...string) error {
return nil
}
// Return cue source for this value
func (v *Value) Source() ([]byte, error) {
return cueformat.Node(v.val.Eval().Syntax())
}
// Return cue source for this value, as a Go string
func (v *Value) SourceString() (string, error) {
b, err := v.Source()
return string(b), err
}
func (v *Value) IsEmptyStruct() bool {
if st, err := v.Struct(); err == nil {
if st.Len() == 0 {