From 795b7f585cd82addf645558c803a01018dac4053 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Jan 2021 16:39:29 -0800 Subject: [PATCH] Clean up client config. Preparation for fixing local dir issues Signed-off-by: Solomon Hykes --- dagger/client.go | 168 +++++++++++++++++++++++++++--------------- dagger/script.go | 3 +- dagger/script_test.go | 4 +- dagger/value_test.go | 2 +- 4 files changed, 112 insertions(+), 65 deletions(-) diff --git a/dagger/client.go b/dagger/client.go index 3737620f..5c13542f 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -31,19 +31,19 @@ const ( bkBootKey = "boot" bkInputKey = "input" - defaultBootDir = "." - - // FIXME: rename to defaultConfig ? - defaultBootScript = ` -bootdir: string | *"." -bootscript: [ - { - do: "local" - dir: bootdir - include: ["*.cue", "cue.mod"] - }, -] -` + // Base client config, for default values & schema validation. + baseClientConfig = ` + close({ + bootdir: string | *"." + boot: [...{do:string,...}] | *[ + { + do: "local" + dir: bootdir + include: ["*.cue", "cue.mod"] + } + ] + }) + ` ) type Client struct { @@ -54,26 +54,33 @@ type Client struct { } type ClientConfig struct { - Host string - Boot string + // Buildkit host address, eg. `docker://buildkitd` + Host string + // Env boot script, eg. `[{do:"local",dir:"."}]` + Boot string + // Env boot dir, eg. `.` + // May be referenced by boot script. BootDir string - Input string + // Input overlay, eg. `www: source: #dagger: compute: [{do:"local",dir:"./src"}]` + Input string } -func NewClient(ctx context.Context, cfg ClientConfig) (*Client, error) { - // buildkit client - if cfg.Host == "" { - cfg.Host = os.Getenv("BUILDKIT_HOST") - } - if cfg.Host == "" { - cfg.Host = defaultBuildkitHost - } - if cfg.Boot == "" { - cfg.Boot = defaultBootScript - } - if cfg.BootDir == "" { - cfg.BootDir = defaultBootDir +func NewClient(ctx context.Context, cfg ClientConfig) (result *Client, err error) { + defer func() { + if err != nil { + // Expand cue errors to get full details + err = cueErr(err) + } + }() + // Finalize config values + localdirs, err := (&cfg).Finalize(ctx) + if err != nil { + return nil, errors.Wrap(err, "client config") } + log.Ctx(ctx).Debug(). + Interface("cfg", cfg). + Interface("localdirs", localdirs). + Msg("finalized client config") c, err := bk.New(ctx, cfg.Host) if err != nil { return nil, errors.Wrap(err, "buildkit client") @@ -81,29 +88,80 @@ func NewClient(ctx context.Context, cfg ClientConfig) (*Client, error) { return &Client{ c: c, cfg: cfg, - localdirs: map[string]string{}, + localdirs: localdirs, }, nil } -func (c *Client) LocalDirs(ctx context.Context) ([]string, error) { - boot, err := c.BootScript() - if err != nil { - return nil, err +// Compile config, fill in final values, +// and return a rollup of local directories +// referenced in the config. +// Localdirs may be referenced in 2 places: +// 1. Boot script +// 2. Input overlay (FIXME: scan not yet implemented) +func (cfg *ClientConfig) Finalize(ctx context.Context) (map[string]string, error) { + localdirs := map[string]string{} + // buildkit client + if cfg.Host == "" { + cfg.Host = os.Getenv("BUILDKIT_HOST") } - return boot.LocalDirs(ctx) + if cfg.Host == "" { + cfg.Host = defaultBuildkitHost + } + // Compile cue template for boot script & boot dir + // (using cue because script may reference dir) + v, err := cfg.Compile() + if err != nil { + return nil, errors.Wrap(err, "invalid client config") + } + // Finalize boot script + boot, err := v.Get("boot").Script() + if err != nil { + return nil, errors.Wrap(err, "invalid env boot script") + } + cfg.Boot = string(boot.Value().JSON()) + // Scan boot script for references to local dirs, to grant access. + bootLocalDirs, err := boot.LocalDirs(ctx) + if err != nil { + return nil, errors.Wrap(err, "scan boot script for local dir access") + } + // Finalize boot dir + cfg.BootDir, err = v.Get("bootdir").String() + if err != nil { + return nil, errors.Wrap(err, "invalid env boot dir") + } + // Scan boot script for references to local dirs, to grant access. + for _, dir := range bootLocalDirs { + // FIXME: randomize local dir references for security + // (currently a malicious cue package may guess common local paths + // and access the corresponding host directory) + localdirs[dir] = dir + } + // FIXME: scan input overlay for references to local dirs, to grant access. + // See issue #41 + return localdirs, nil } -func (c *Client) BootScript() (*Script, error) { +// Compile client config to a cue value +// FIXME: include host and input. +func (cfg ClientConfig) Compile() (v *Value, err error) { cc := &Compiler{} - src, err := cc.Compile("boot.cue", c.cfg.Boot) + v, err = cc.Compile("client.cue", baseClientConfig) if err != nil { - return nil, errors.Wrap(err, "compile") + return nil, errors.Wrap(err, "base client config") } - src, err = src.MergeTarget(c.cfg.BootDir, "bootdir") - if err != nil { - return nil, err + if cfg.BootDir != "" { + v, err = v.Merge(cfg.BootDir, "bootdir") + if err != nil { + return nil, errors.Wrap(err, "client config key 'bootdir'") + } } - return src.Get("bootscript").Script() + if cfg.Boot != "" { + v, err = v.Merge(cfg.Boot, "boot") + if err != nil { + return nil, errors.Wrap(err, "client config key 'boot'") + } + } + return v, nil } func (c *Client) Compute(ctx context.Context) (*Value, error) { @@ -166,22 +224,13 @@ func (c *Client) Compute(ctx context.Context) (*Value, error) { func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.WriteCloser) error { lg := log.Ctx(ctx) - boot, err := c.BootScript() - if err != nil { - return errors.Wrap(err, "assemble boot script") - } - bootSource, err := boot.Value().Source() - if err != nil { - return errors.Wrap(err, "serialize boot script") - } - lg.Debug().Bytes("bootSource", bootSource).Msg("assembled boot script") // Setup solve options opts := bk.SolveOpt{ FrontendAttrs: map[string]string{ bkInputKey: c.cfg.Input, - bkBootKey: string(bootSource), + bkBootKey: c.cfg.Boot, }, - LocalDirs: map[string]string{}, + LocalDirs: c.localdirs, // FIXME: catch output & return as cue value Exports: []bk.ExportEntry{ { @@ -192,15 +241,12 @@ func (c *Client) buildfn(ctx context.Context, ch chan *bk.SolveStatus, w io.Writ }, }, } - // Connect local dirs - localdirs, err := c.LocalDirs(ctx) - if err != nil { - return errors.Wrap(err, "connect local dirs") - } - for _, dir := range localdirs { - opts.LocalDirs[dir] = dir - } // Call buildkit solver + lg.Debug(). + Interface("localdirs", opts.LocalDirs). + Interface("attrs", opts.FrontendAttrs). + Interface("host", c.cfg.Host). + Msg("spawning buildkit job") resp, err := c.c.Build(ctx, opts, "", Compute, ch) if err != nil { return errors.Wrap(bkCleanError(err), "buildkit solve") diff --git a/dagger/script.go b/dagger/script.go index fd566f6b..158df8fa 100644 --- a/dagger/script.go +++ b/dagger/script.go @@ -73,11 +73,12 @@ func (s *Script) LocalDirs(ctx context.Context) ([]string, error) { var dirs []string err := s.Walk(ctx, func(op *Op) error { if err := op.Validate("#Local"); err != nil { + // Ignore all operations except 'do:"local"' return nil } dir, err := op.Get("dir").String() if err != nil { - return err + return errors.Wrap(err, "invalid 'local' operation") } dirs = append(dirs, dir) return nil diff --git a/dagger/script_test.go b/dagger/script_test.go index ccc192cc..a6f3b14a 100644 --- a/dagger/script_test.go +++ b/dagger/script_test.go @@ -85,11 +85,11 @@ func TestWalkBootScript(t *testing.T) { ctx := context.TODO() cc := &Compiler{} - cfg, err := cc.Compile("clientconfig.cue", defaultBootScript) + cfg, err := cc.Compile("clientconfig.cue", baseClientConfig) if err != nil { t.Fatal(err) } - script, err := cfg.Get("bootscript").Script() + script, err := cfg.Get("boot").Script() if err != nil { t.Fatal(err) } diff --git a/dagger/value_test.go b/dagger/value_test.go index 788276f5..92e8b2e4 100644 --- a/dagger/value_test.go +++ b/dagger/value_test.go @@ -31,7 +31,7 @@ func TestJSON(t *testing.T) { func TestCompileBootScript(t *testing.T) { cc := &Compiler{} - cfg, err := cc.Compile("boot.cue", defaultBootScript) + cfg, err := cc.Compile("boot.cue", baseClientConfig) if err != nil { t.Fatal(err) }