From 0aea10d23e1cd895b32a63bc2eb802ad203b5869 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 30 Nov 2021 12:48:09 -0800 Subject: [PATCH] dagger.#FS support - Implement dagger.#FS support - Migrate `context.imports` to dagger.#FS - Backward compat: dagger.#FS can be passed in lieu of a dagger.#Artifact - For instance, an import (`dagger.#FS`) can be passed to the current `yarn.#Package` implementation Signed-off-by: Andrea Luzzardi --- client/client.go | 11 +++-- cmd/dagger/cmd/compute.go | 2 +- cmd/dagger/cmd/edit.go | 2 +- cmd/dagger/cmd/input/list.go | 2 +- cmd/dagger/cmd/input/root.go | 2 +- cmd/dagger/cmd/output/list.go | 2 +- cmd/dagger/cmd/up.go | 8 +--- compiler/compiler.go | 12 +++++ docs/reference/dagger/README.md | 14 +++++- environment/pipeline.go | 83 +++++++++++++++++++++++++++------ plan/plan.go | 31 ++++++------ plan/task/import.go | 67 +++++++++++++++++++++----- plan/task/secretenv.go | 9 ++-- plan/task/secretfile.go | 9 ++-- plan/task/util.go | 14 ++++++ plancontext/context.go | 16 ++++--- plancontext/directory.go | 54 --------------------- plancontext/fs.go | 32 +++++++++++++ plancontext/localdir.go | 60 ++++++++++++++++++++++++ state/input.go | 39 +++++++++++++--- stdlib/dagger/dagger.cue | 17 ++++++- stdlib/dagger/op/op.cue | 6 ++- stdlib/dagger/plan.cue | 2 +- 23 files changed, 359 insertions(+), 135 deletions(-) create mode 100644 plan/task/util.go delete mode 100644 plancontext/directory.go create mode 100644 plancontext/fs.go create mode 100644 plancontext/localdir.go diff --git a/client/client.go b/client/client.go index 8569d9cd..89fb6139 100644 --- a/client/client.go +++ b/client/client.go @@ -75,7 +75,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) { type DoFunc func(context.Context, solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, localdirs map[string]string, fn DoFunc) error { +func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) @@ -90,13 +90,13 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, localdirs ma // Spawn build function eg.Go(func() error { - return c.buildfn(gctx, pctx, localdirs, fn, events) + return c.buildfn(gctx, pctx, fn, events) }) return eg.Wait() } -func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, localdirs map[string]string, fn DoFunc, ch chan *bk.SolveStatus) error { +func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error { wg := sync.WaitGroup{} // Close output channel @@ -111,6 +111,11 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, localdi // buildkit auth provider (registry) auth := solver.NewRegistryAuthProvider() + localdirs, err := pctx.LocalDirs.Paths() + if err != nil { + return err + } + // Setup solve options opts := bk.SolveOpt{ LocalDirs: localdirs, diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index bb942a8d..320d87be 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -198,7 +198,7 @@ var computeCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("unable to create environment") } - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set checkInputs(ctx, env) diff --git a/cmd/dagger/cmd/edit.go b/cmd/dagger/cmd/edit.go index 6f9bfcf5..a80a2343 100644 --- a/cmd/dagger/cmd/edit.go +++ b/cmd/dagger/cmd/edit.go @@ -83,7 +83,7 @@ var editCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 712140f5..e66aac28 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -47,7 +47,7 @@ var listCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { inputs, err := env.ScanInputs(ctx, false) if err != nil { return err diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index ff2f10ac..57b1a785 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -59,7 +59,7 @@ func updateEnvironmentInput(ctx context.Context, cmd *cobra.Command, target stri } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // the inputs are set, check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/output/list.go b/cmd/dagger/cmd/output/list.go index ffcb0dad..36887693 100644 --- a/cmd/dagger/cmd/output/list.go +++ b/cmd/dagger/cmd/output/list.go @@ -46,7 +46,7 @@ var listCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { return ListOutputs(ctx, env, true) }) diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index cffd0648..519e39b8 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -80,7 +80,7 @@ var upCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("unable to create environment") } - err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set if err := checkInputs(ctx, env); err != nil { return err @@ -119,11 +119,7 @@ func europaUp(ctx context.Context, cl *client.Client, path string) error { lg.Fatal().Err(err).Msg("failed to load plan") } - localdirs, err := p.LocalDirectories() - if err != nil { - return err - } - return cl.Do(ctx, p.Context(), localdirs, func(ctx context.Context, s solver.Solver) error { + return cl.Do(ctx, p.Context(), func(ctx context.Context, s solver.Solver) error { if err := p.Up(ctx, s); err != nil { return err } diff --git a/compiler/compiler.go b/compiler/compiler.go index 0f3c5784..1dd4b7c8 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -24,6 +24,10 @@ func NewValue() *Value { return DefaultCompiler.NewValue() } +func NewValueWithContent(x interface{}, selectors ...cue.Selector) (*Value, error) { + return DefaultCompiler.NewValueWithContent(x, selectors...) +} + // FIXME can be refactored away now? func Wrap(v cue.Value) *Value { return DefaultCompiler.Wrap(v) @@ -80,6 +84,14 @@ func (c *Compiler) NewValue() *Value { return empty } +func (c *Compiler) NewValueWithContent(x interface{}, selectors ...cue.Selector) (*Value, error) { + v := c.NewValue() + if err := v.FillPath(cue.MakePath(selectors...), x); err != nil { + return nil, err + } + return v, nil +} + func (c *Compiler) Compile(name string, src string) (*Value, error) { c.lock() defer c.unlock() diff --git a/docs/reference/dagger/README.md b/docs/reference/dagger/README.md index 779f95f0..3f545bc6 100644 --- a/docs/reference/dagger/README.md +++ b/docs/reference/dagger/README.md @@ -20,6 +20,18 @@ _No input._ _No output._ +## dagger.#FS + +A reference to a filesystem tree. For example: - The root filesystem of a container - A source code repository - A directory containing binary artifacts Rule of thumb: if it fits in a tar archive, it fits in a #FS. + +### dagger.#FS Inputs + +_No input._ + +### dagger.#FS Outputs + +_No output._ + ## dagger.#Plan A deployment plan executed by `dagger up` @@ -34,7 +46,7 @@ _No output._ ## dagger.#Secret -Secret value +A reference to an external secret, for example: - A password - A SSH private key - An API token Secrets are never merged in the Cue tree. They can only be used by a special filesystem mount designed to minimize leak risk. ### dagger.#Secret Inputs diff --git a/environment/pipeline.go b/environment/pipeline.go index 34c839b1..e1155891 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -40,6 +40,13 @@ const ( StateCompleted = State("completed") ) +var ( + fsIDPath = cue.MakePath( + cue.Hid("_fs", "alpha.dagger.io/dagger"), + cue.Str("id"), + ) +) + // An execution pipeline type Pipeline struct { code *compiler.Value @@ -95,8 +102,19 @@ func IsComponent(v *compiler.Value) bool { return v.Lookup("#up").Exists() } +func isFS(v *compiler.Value) bool { + return v.LookupPath(fsIDPath).Exists() +} + func ops(code *compiler.Value) ([]*compiler.Value, error) { ops := []*compiler.Value{} + + // dagger.#FS forward compat + // FIXME: remove this + if isFS(code) { + ops = append(ops, code) + } + // 1. attachment array if IsComponent(code) { xops, err := code.Lookup("#up").List() @@ -138,6 +156,12 @@ func Analyze(fn func(*compiler.Value) error, code *compiler.Value) error { } func analyzeOp(fn func(*compiler.Value) error, op *compiler.Value) error { + // dagger.#FS forward compat + // FIXME: remove this + if isFS(op) { + return nil + } + if err := fn(op); err != nil { return err } @@ -243,6 +267,21 @@ func (p *Pipeline) run(ctx context.Context) error { } func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { + // dagger.#FS forward compat + // FIXME: remove this + if isFS(op) { + id, err := op.LookupPath(fsIDPath).String() + if err != nil { + return st, err + } + + fs := p.pctx.FS.Get(plancontext.ContextKey(id)) + if fs == nil { + return st, fmt.Errorf("fs %q not found", id) + } + return fs.Result.ToState() + } + do, err := op.Lookup("do").String() if err != nil { return st, err @@ -361,31 +400,49 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) ( } func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { - id, err := op.Lookup("id").String() + dir, err := op.Lookup("dir").String() if err != nil { return st, err } - dir := p.pctx.Directories.Get(plancontext.ContextKey(id)) - if dir == nil { - return st, fmt.Errorf("directory %q not found", id) - } opts := []llb.LocalOption{ - llb.WithCustomName(p.vertexNamef("Local %s", dir.Path)), + llb.WithCustomName(p.vertexNamef("Local %s", dir)), // Without hint, multiple `llb.Local` operations on the // same path get a different digest. llb.SessionID(p.s.SessionID()), - llb.SharedKeyHint(dir.Path), + llb.SharedKeyHint(dir), } - if len(dir.Include) > 0 { - opts = append(opts, llb.IncludePatterns(dir.Include)) + includes, err := op.Lookup("include").List() + if err != nil { + return st, err + } + if len(includes) > 0 { + includePatterns := []string{} + for _, i := range includes { + pattern, err := i.String() + if err != nil { + return st, err + } + includePatterns = append(includePatterns, pattern) + } + opts = append(opts, llb.IncludePatterns(includePatterns)) } + excludes, err := op.Lookup("exclude").List() + if err != nil { + return st, err + } // Excludes .dagger directory by default excludePatterns := []string{"**/.dagger/"} - if len(dir.Exclude) > 0 { - excludePatterns = dir.Exclude + if len(excludes) > 0 { + for _, i := range excludes { + pattern, err := i.String() + if err != nil { + return st, err + } + excludePatterns = append(excludePatterns, pattern) + } } opts = append(opts, llb.ExcludePatterns(excludePatterns)) @@ -396,13 +453,13 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) return st.File( llb.Copy( llb.Local( - dir.Path, + dir, opts..., ), "/", "/", ), - llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir.Path)), + llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)), ), nil } diff --git a/plan/plan.go b/plan/plan.go index 49f5a455..f41364c4 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -3,7 +3,6 @@ package plan import ( "context" "fmt" - "path/filepath" "strings" "time" @@ -35,10 +34,16 @@ func Load(ctx context.Context, path, pkg string) (*Plan, error) { return nil, err } - return &Plan{ + p := &Plan{ context: plancontext.New(), source: v, - }, nil + } + + if err := p.registerLocalDirs(); err != nil { + return nil, err + } + + return p, nil } func (p *Plan) Context() *plancontext.Context { @@ -49,30 +54,26 @@ func (p *Plan) Source() *compiler.Value { return p.source } -// LocalDirectories scans the context for local imports. +// registerLocalDirectories scans the context for local imports. // BuildKit requires to known the list of directories ahead of time. -func (p *Plan) LocalDirectories() (map[string]string, error) { - dirs := map[string]string{} - +func (p *Plan) registerLocalDirs() error { imports, err := p.source.Lookup("context.imports").Fields() if err != nil { - return nil, err + return err } for _, v := range imports { dir, err := v.Value.Lookup("path").String() if err != nil { - return nil, err - } - abs, err := filepath.Abs(dir) - if err != nil { - return nil, err + return err } - dirs[dir] = abs + p.context.LocalDirs.Register(&plancontext.LocalDir{ + Path: dir, + }) } - return dirs, nil + return nil } // Up executes the plan diff --git a/plan/task/import.go b/plan/task/import.go index 40b4e2f9..5f54e5e6 100644 --- a/plan/task/import.go +++ b/plan/task/import.go @@ -2,9 +2,9 @@ package task import ( "context" - "fmt" - "os" + "cuelang.org/go/cue" + "github.com/moby/buildkit/client/llb" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" @@ -17,21 +17,64 @@ func init() { type importTask struct { } -func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { - var dir *plancontext.Directory +func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + var dir struct { + Path string + Include []string + Exclude []string + } if err := v.Decode(&dir); err != nil { return nil, err } - // Check that directory exists - if _, err := os.Stat(dir.Path); os.IsNotExist(err) { - return nil, fmt.Errorf("%q dir doesn't exist", dir.Path) + opts := []llb.LocalOption{ + withCustomName(v, "Local %s", dir.Path), + // Without hint, multiple `llb.Local` operations on the + // same path get a different digest. + llb.SessionID(s.SessionID()), + llb.SharedKeyHint(dir.Path), } - id := pctx.Directories.Register(dir) - return compiler.Compile("", fmt.Sprintf( - `fs: #up: [{do: "local", id: %q}]`, - id, - )) + if len(dir.Include) > 0 { + opts = append(opts, llb.IncludePatterns(dir.Include)) + } + + // Excludes .dagger directory by default + excludePatterns := []string{"**/.dagger/"} + if len(dir.Exclude) > 0 { + excludePatterns = dir.Exclude + } + opts = append(opts, llb.ExcludePatterns(excludePatterns)) + + // FIXME: Remove the `Copy` and use `Local` directly. + // + // Copy'ing is a costly operation which should be unnecessary. + // However, using llb.Local directly breaks caching sometimes for unknown reasons. + st := llb.Scratch().File( + llb.Copy( + llb.Local( + dir.Path, + opts..., + ), + "/", + "/", + ), + withCustomName(v, "Local %s [copy]", dir.Path), + ) + + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + id := pctx.FS.Register(&plancontext.FS{ + Result: result, + }) + + return compiler.NewValueWithContent(id, + cue.Str("fs"), + cue.Hid("_fs", "alpha.dagger.io/dagger"), + cue.Str("id"), + ) } diff --git a/plan/task/secretenv.go b/plan/task/secretenv.go index 11ee5d71..6ec3c821 100644 --- a/plan/task/secretenv.go +++ b/plan/task/secretenv.go @@ -40,9 +40,8 @@ func (c secretEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ sol PlainText: env, }) - out := compiler.NewValue() - if err := out.FillPath(cue.ParsePath("contents.id"), id); err != nil { - return nil, err - } - return out, nil + return compiler.NewValueWithContent(id, + cue.Str("contents"), + cue.Str("id"), + ) } diff --git a/plan/task/secretfile.go b/plan/task/secretfile.go index 5d38cd53..e75d056d 100644 --- a/plan/task/secretfile.go +++ b/plan/task/secretfile.go @@ -39,9 +39,8 @@ func (c secretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ so PlainText: string(data), }) - out := compiler.NewValue() - if err := out.FillPath(cue.ParsePath("contents.id"), id); err != nil { - return nil, err - } - return out, nil + return compiler.NewValueWithContent(id, + cue.Str("contents"), + cue.Str("id"), + ) } diff --git a/plan/task/util.go b/plan/task/util.go new file mode 100644 index 00000000..8c8b66ab --- /dev/null +++ b/plan/task/util.go @@ -0,0 +1,14 @@ +package task + +import ( + "fmt" + + "github.com/moby/buildkit/client/llb" + "go.dagger.io/dagger/compiler" +) + +func withCustomName(v *compiler.Value, format string, a ...interface{}) llb.ConstraintsOpt { + prefix := fmt.Sprintf("@%s@", v.Path().String()) + name := fmt.Sprintf(format, a...) + return llb.WithCustomName(prefix + " " + name) +} diff --git a/plancontext/context.go b/plancontext/context.go index 743756d0..2efa74cf 100644 --- a/plancontext/context.go +++ b/plancontext/context.go @@ -15,10 +15,11 @@ type ContextKey string // id := ctx.Secrets.Register("mysecret") // secret := ctx.Secrets.Get(id) type Context struct { - Platform *platformContext - Directories *directoryContext - Secrets *secretContext - Services *serviceContext + Platform *platformContext + FS *fsContext + LocalDirs *localDirContext + Secrets *secretContext + Services *serviceContext } func New() *Context { @@ -26,8 +27,11 @@ func New() *Context { Platform: &platformContext{ platform: defaultPlatform, }, - Directories: &directoryContext{ - store: make(map[ContextKey]*Directory), + FS: &fsContext{ + store: make(map[ContextKey]*FS), + }, + LocalDirs: &localDirContext{ + store: make(map[ContextKey]*LocalDir), }, Secrets: &secretContext{ store: make(map[ContextKey]*Secret), diff --git a/plancontext/directory.go b/plancontext/directory.go deleted file mode 100644 index 3011f682..00000000 --- a/plancontext/directory.go +++ /dev/null @@ -1,54 +0,0 @@ -package plancontext - -import "sync" - -type Directory struct { - Path string - Include []string - Exclude []string -} - -type directoryContext struct { - l sync.RWMutex - store map[ContextKey]*Directory -} - -func (c *directoryContext) Register(directory *Directory) ContextKey { - c.l.Lock() - defer c.l.Unlock() - - id := hashID(directory) - c.store[id] = directory - return id -} - -func (c *directoryContext) Get(id ContextKey) *Directory { - c.l.RLock() - defer c.l.RUnlock() - - return c.store[id] -} - -func (c *directoryContext) List() []*Directory { - c.l.RLock() - defer c.l.RUnlock() - - directories := make([]*Directory, 0, len(c.store)) - for _, d := range c.store { - directories = append(directories, d) - } - - return directories -} - -func (c *directoryContext) Paths() map[string]string { - c.l.RLock() - defer c.l.RUnlock() - - directories := make(map[string]string) - for _, d := range c.store { - directories[d.Path] = d.Path - } - - return directories -} diff --git a/plancontext/fs.go b/plancontext/fs.go new file mode 100644 index 00000000..7a8b1267 --- /dev/null +++ b/plancontext/fs.go @@ -0,0 +1,32 @@ +package plancontext + +import ( + "sync" + + bkgw "github.com/moby/buildkit/frontend/gateway/client" +) + +type FS struct { + Result bkgw.Reference +} + +type fsContext struct { + l sync.RWMutex + store map[ContextKey]*FS +} + +func (c *fsContext) Register(fs *FS) ContextKey { + c.l.Lock() + defer c.l.Unlock() + + id := hashID(fs) + c.store[id] = fs + return id +} + +func (c *fsContext) Get(id ContextKey) *FS { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[id] +} diff --git a/plancontext/localdir.go b/plancontext/localdir.go new file mode 100644 index 00000000..b2ea9494 --- /dev/null +++ b/plancontext/localdir.go @@ -0,0 +1,60 @@ +package plancontext + +import ( + "path/filepath" + "sync" +) + +type LocalDir struct { + Path string +} + +type localDirContext struct { + l sync.RWMutex + store map[ContextKey]*LocalDir +} + +func (c *localDirContext) Register(directory *LocalDir) ContextKey { + c.l.Lock() + defer c.l.Unlock() + + id := hashID(directory) + c.store[id] = directory + return id +} + +func (c *localDirContext) Get(id ContextKey) *LocalDir { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[id] +} + +func (c *localDirContext) List() []*LocalDir { + c.l.RLock() + defer c.l.RUnlock() + + directories := make([]*LocalDir, 0, len(c.store)) + for _, d := range c.store { + directories = append(directories, d) + } + + return directories +} + +func (c *localDirContext) Paths() (map[string]string, error) { + c.l.RLock() + defer c.l.RUnlock() + + directories := make(map[string]string) + for _, d := range c.store { + abs, err := filepath.Abs(d.Path) + if err != nil { + return nil, err + } + + directories[d.Path] = abs + } + + return directories, nil +} diff --git a/state/input.go b/state/input.go index 6a2ccf42..48f561b2 100644 --- a/state/input.go +++ b/state/input.go @@ -1,6 +1,7 @@ package state import ( + "encoding/json" "fmt" "io/ioutil" "os" @@ -82,6 +83,27 @@ type dirInput struct { } func (dir dirInput) Compile(state *State) (*compiler.Value, error) { + // FIXME: serialize an intermediate struct, instead of generating cue source + + // json.Marshal([]string{}) returns []byte("null"), which wreaks havoc + // in Cue because `null` is not a `[...string]` + includeLLB := []byte("[]") + if len(dir.Include) > 0 { + var err error + includeLLB, err = json.Marshal(dir.Include) + if err != nil { + return nil, err + } + } + excludeLLB := []byte("[]") + if len(dir.Exclude) > 0 { + var err error + excludeLLB, err = json.Marshal(dir.Exclude) + if err != nil { + return nil, err + } + } + p := dir.Path if !filepath.IsAbs(p) { p = filepath.Clean(path.Join(state.Project, dir.Path)) @@ -94,15 +116,20 @@ func (dir dirInput) Compile(state *State) (*compiler.Value, error) { return nil, fmt.Errorf("%q dir doesn't exist", dir.Path) } - id := state.Context.Directories.Register(&plancontext.Directory{ - Path: p, - Include: dir.Include, - Exclude: dir.Exclude, + dirPath, err := json.Marshal(p) + if err != nil { + return nil, err + } + + state.Context.LocalDirs.Register(&plancontext.LocalDir{ + Path: p, }) llb := fmt.Sprintf( - `#up: [{do:"local", id: "%s"}]`, - id, + `#up: [{do: "local", dir: %s, include: %s, exclude: %s}]`, + dirPath, + includeLLB, + excludeLLB, ) return compiler.Compile("", llb) } diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 01a1944b..c37f6621 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -5,6 +5,16 @@ import ( "alpha.dagger.io/dagger/op" ) +// A reference to a filesystem tree. +// For example: +// - The root filesystem of a container +// - A source code repository +// - A directory containing binary artifacts +// Rule of thumb: if it fits in a tar archive, it fits in a #FS. +#FS: { + _fs: id: string +} + // An artifact such as source code checkout, container image, binary archive... // May be passed as user input, or computed by a buildkit pipeline #Artifact: { @@ -21,7 +31,12 @@ import ( id: string } -// Secret value +// A reference to an external secret, for example: +// - A password +// - A SSH private key +// - An API token +// Secrets are never merged in the Cue tree. They can only be used +// by a special filesystem mount designed to minimize leak risk. #Secret: { @dagger(secret) diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index 6587186c..cf4d62c0 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -20,8 +20,10 @@ package op } #Local: { - do: "local" - id: string + do: "local" + dir: string + include: [...string] + exclude: [...string] } // FIXME: bring back load (more efficient than copy) diff --git a/stdlib/dagger/plan.cue b/stdlib/dagger/plan.cue index ec71aa9b..fff7cc68 100644 --- a/stdlib/dagger/plan.cue +++ b/stdlib/dagger/plan.cue @@ -20,7 +20,7 @@ package dagger path: string include?: [...string] exclude?: [...string] - fs: #Artifact + fs: #FS } // Securely load external secrets