From a61e8dcb627afe650f922e5e1e1e91c4f181ecf9 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 16 Nov 2021 16:13:45 -0800 Subject: [PATCH 1/3] prepare the transition to `#Plan.context` This change helps the transition between `dagger input` and `#Plan.context`. In summary, the codebase now relies on a *context* for execution with mapping to *IDs*. In the future, *context* will come from a `#Plan.context`. In the meantime, a bridge converts `dagger input` to a plan context. This allows both *old* and *new* style configurations to co-exist with the same underlying engine. - Implement `plancontext`. Context holds the execution context for a plan. Currently this includes the platform, local directories, secrets and services (e.g. unix/npipe). - Contextual data can be registered at any point. In the future, this will be done by `#Plan.context` - Migrated the `dagger input` codebase to register inputs in a `plancontext` - Migrated low-level types/operations to the *Context ID* pattern. - `dagger.#Stream` now only includes an `id` (instead of `unix` path) - `dagger.#Secret` still includes only an ID, but now it's based off `plancontext` - `op.#Local` now only includes an `id` (instead of `path`, `include`, `exclude`. Signed-off-by: Andrea Luzzardi --- client/client.go | 76 ++++++------------ cmd/dagger/cmd/compute.go | 9 ++- cmd/dagger/cmd/edit.go | 7 +- cmd/dagger/cmd/input/container.go | 33 -------- cmd/dagger/cmd/input/list.go | 9 ++- cmd/dagger/cmd/input/root.go | 11 ++- cmd/dagger/cmd/output/list.go | 7 +- cmd/dagger/cmd/up.go | 7 +- environment/environment.go | 87 ++------------------- environment/pipeline.go | 124 +++++++++++------------------- plancontext/context.go | 48 ++++++++++++ plancontext/directory.go | 42 ++++++++++ plancontext/platform.go | 32 ++++++++ plancontext/secret.go | 40 ++++++++++ plancontext/service.go | 29 +++++++ solver/secretsprovider.go | 42 ++-------- solver/socketprovider.go | 26 +++---- solver/socketprovider_unix.go | 14 ++-- solver/socketprovider_windows.go | 13 ++-- solver/solver.go | 17 ++-- state/input.go | 117 +++++++++------------------- state/project.go | 9 +++ state/state.go | 7 +- stdlib/dagger/dagger.cue | 8 +- stdlib/dagger/op/op.cue | 6 +- stdlib/docker/docker.cue | 3 + tests/core.bats | 1 - 27 files changed, 401 insertions(+), 423 deletions(-) delete mode 100644 cmd/dagger/cmd/input/container.go create mode 100644 plancontext/context.go create mode 100644 plancontext/directory.go create mode 100644 plancontext/platform.go create mode 100644 plancontext/secret.go create mode 100644 plancontext/service.go diff --git a/client/client.go b/client/client.go index 9667c6f7..8a6f189b 100644 --- a/client/client.go +++ b/client/client.go @@ -8,7 +8,6 @@ import ( "sync" "github.com/containerd/containerd/platforms" - "go.opentelemetry.io/otel" "golang.org/x/sync/errgroup" "github.com/rs/zerolog/log" @@ -23,13 +22,12 @@ import ( "github.com/moby/buildkit/session" // docker output + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/util/buildkitd" "go.dagger.io/dagger/util/progressui" "go.dagger.io/dagger/compiler" - "go.dagger.io/dagger/environment" "go.dagger.io/dagger/solver" - "go.dagger.io/dagger/state" ) // Client is a dagger client @@ -74,36 +72,31 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) { }, nil } -type DoFunc func(context.Context, *environment.Environment, solver.Solver) error +type DoFunc func(context.Context, solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error { +func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) - env, err := environment.New(state) - if err != nil { - return err - } - // Spawn print function events := make(chan *bk.SolveStatus) eg.Go(func() error { // Create a background context so that logging will not be cancelled // with the main context. dispCtx := lg.WithContext(context.Background()) - return c.logSolveStatus(dispCtx, state, events) + return c.logSolveStatus(dispCtx, pctx, events) }) // Spawn build function eg.Go(func() error { - return c.buildfn(gctx, state, env, fn, events) + return c.buildfn(gctx, pctx, fn, events) }) return eg.Wait() } -func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.Environment, fn DoFunc, ch chan *bk.SolveStatus) error { +func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error { wg := sync.WaitGroup{} // Close output channel @@ -115,25 +108,21 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment. lg := log.Ctx(ctx) - // Scan local dirs to grant access - localdirs, err := env.LocalDirs() - if err != nil { - return err - } - // buildkit auth provider (registry) auth := solver.NewRegistryAuthProvider() - // session (secrets & store) - secretsStore := solver.NewSecretsStoreProvider(st) + localdirs := map[string]string{} + for _, dir := range pctx.Directories.List() { + localdirs[dir.Path] = dir.Path + } // Setup solve options opts := bk.SolveOpt{ LocalDirs: localdirs, Session: []session.Attachable{ auth, - secretsStore.Secrets, - solver.NewDockerSocketProvider(), + solver.NewSecretsStoreProvider(pctx), + solver.NewDockerSocketProvider(pctx), }, CacheExports: c.cfg.CacheExports, CacheImports: c.cfg.CacheImports, @@ -167,12 +156,11 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment. resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { s := solver.New(solver.Opts{ - Control: c.c, - Gateway: gw, - Events: eventsCh, - Auth: auth, - SecretsStore: secretsStore, - NoCache: c.cfg.NoCache, + Control: c.c, + Gateway: gw, + Events: eventsCh, + Auth: auth, + NoCache: c.cfg.NoCache, }) // Close events channel @@ -180,28 +168,12 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment. // Compute output overlay if fn != nil { - if err := fn(ctx, env, s); err != nil { + if err := fn(ctx, s); err != nil { return nil, compiler.Err(err) } } - // Export environment to a cue directory - // FIXME: this should be elsewhere - lg.Debug().Msg("exporting environment") - - tr := otel.Tracer("client") - _, span := tr.Start(ctx, "environment.Export") - defer span.End() - - computed := env.Computed().JSON().PrettyString() - st := llb. - Scratch(). - File( - llb.Mkfile("computed.json", 0600, []byte(computed)), - llb.WithCustomName("[internal] serializing computed values"), - ) - - ref, err := s.Solve(ctx, st, platforms.DefaultSpec()) + ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec()) if err != nil { return nil, err } @@ -224,7 +196,7 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment. return nil } -func (c *Client) logSolveStatus(ctx context.Context, st *state.State, ch chan *bk.SolveStatus) error { +func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, ch chan *bk.SolveStatus) error { parseName := func(v *bk.Vertex) (string, string) { // Pattern: `@name@ message`. Minimal length is len("@X@ ") if len(v.Name) < 2 || !strings.HasPrefix(v.Name, "@") { @@ -241,13 +213,11 @@ func (c *Client) logSolveStatus(ctx context.Context, st *state.State, ch chan *b } // Just like sprintf, but redacts secrets automatically + secrets := pctx.Secrets.List() secureSprintf := func(format string, a ...interface{}) string { s := fmt.Sprintf(format, a...) - for _, i := range st.Inputs { - if i.Secret == nil { - continue - } - s = strings.ReplaceAll(s, i.Secret.PlainText(), "***") + for _, secret := range secrets { + s = strings.ReplaceAll(s, secret.PlainText, "***") } return s } diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index d355ca06..b3e4ab48 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -15,6 +15,7 @@ import ( "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" "go.mozilla.org/sops/v3" @@ -43,6 +44,7 @@ var computeCmd = &cobra.Command{ doneCh := common.TrackCommand(ctx, cmd) st := &state.State{ + Context: plancontext.New(), Name: "FIXME", Platform: platforms.Format(specs.Platform{OS: "linux", Architecture: "amd64"}), Path: args[0], @@ -191,7 +193,12 @@ var computeCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("failed to compile inputs") } - err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set checkInputs(ctx, env) diff --git a/cmd/dagger/cmd/edit.go b/cmd/dagger/cmd/edit.go index d9b885df..3a229e79 100644 --- a/cmd/dagger/cmd/edit.go +++ b/cmd/dagger/cmd/edit.go @@ -77,8 +77,13 @@ var editCmd = &cobra.Command{ st.Plan = newState.Plan st.Inputs = newState.Inputs + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + cl := common.NewClient(ctx) - err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/input/container.go b/cmd/dagger/cmd/input/container.go deleted file mode 100644 index a9e4b994..00000000 --- a/cmd/dagger/cmd/input/container.go +++ /dev/null @@ -1,33 +0,0 @@ -package input - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" - "go.dagger.io/dagger/cmd/dagger/logger" - "go.dagger.io/dagger/state" -) - -var containerCmd = &cobra.Command{ - Use: "container TARGET CONTAINER-IMAGE", - Short: "Add a container image as input artifact", - Args: cobra.ExactArgs(2), - PreRun: func(cmd *cobra.Command, args []string) { - // Fix Viper bug for duplicate flags: - // https://github.com/spf13/viper/issues/233 - if err := viper.BindPFlags(cmd.Flags()); err != nil { - panic(err) - } - }, - Run: func(cmd *cobra.Command, args []string) { - lg := logger.New() - ctx := lg.WithContext(cmd.Context()) - - updateEnvironmentInput(ctx, cmd, args[0], state.DockerInput(args[1])) - }, -} - -func init() { - if err := viper.BindPFlags(containerCmd.Flags()); err != nil { - panic(err) - } -} diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 8e876331..5661438d 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -41,8 +41,13 @@ var listCmd = &cobra.Command{ doneCh := common.TrackProjectCommand(ctx, cmd, project, st) - c := common.NewClient(ctx) - err := c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + + cl := common.NewClient(ctx) + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { inputs, err := env.ScanInputs(ctx, false) if err != nil { return err diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index 391f5321..b52f6c8a 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -25,7 +25,6 @@ func init() { Cmd.AddCommand( dirCmd, gitCmd, - containerCmd, secretCmd, textCmd, jsonCmd, @@ -52,11 +51,15 @@ func updateEnvironmentInput(ctx context.Context, cmd *cobra.Command, target stri Value: target, }) - cl := common.NewClient(ctx) - st.SetInput(target, input) - err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + + cl := common.NewClient(ctx) + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // the inputs are set, check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/output/list.go b/cmd/dagger/cmd/output/list.go index 1567a854..3b910bda 100644 --- a/cmd/dagger/cmd/output/list.go +++ b/cmd/dagger/cmd/output/list.go @@ -40,8 +40,13 @@ var listCmd = &cobra.Command{ doneCh := common.TrackProjectCommand(ctx, cmd, project, st) + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + cl := common.NewClient(ctx) - err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { return ListOutputs(ctx, env, true) }) diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 8dcbcefb..1e8cd38e 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -61,7 +61,12 @@ var upCmd = &cobra.Command{ cl := common.NewClient(ctx) - err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + env, err := environment.New(st) + if err != nil { + lg.Fatal().Msg("unable to create environment") + } + + err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set if err := checkInputs(ctx, env); err != nil { return err diff --git a/environment/environment.go b/environment/environment.go index 50dc5ca5..cd290b54 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -3,13 +3,11 @@ package environment import ( "context" "fmt" - "path/filepath" "cuelang.org/go/cue" cueflow "cuelang.org/go/tools/flow" - "github.com/containerd/containerd/platforms" - specs "github.com/opencontainers/image-spec/specs-go/v1" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" @@ -74,59 +72,8 @@ func (e *Environment) Computed() *compiler.Value { return e.computed } -// Scan all scripts in the environment for references to local directories (do:"local"), -// and return all referenced directory names. -// This is used by clients to grant access to local directories when they are referenced -// by user-specified scripts. -func (e *Environment) LocalDirs() (map[string]string, error) { - dirs := map[string]string{} - - localdirs := func(code *compiler.Value) error { - return Analyze( - func(op *compiler.Value) error { - do, err := op.Lookup("do").String() - if err != nil { - return err - } - if do != "local" { - return nil - } - dir, err := op.Lookup("dir").String() - if err != nil { - return err - } - abs, err := filepath.Abs(dir) - if err != nil { - return err - } - - dirs[dir] = abs - return nil - }, - code, - ) - } - // 1. Scan the environment state - // FIXME: use a common `flow` instance to avoid rescanning the tree. - src := compiler.NewValue() - if err := src.FillPath(cue.MakePath(), e.plan); err != nil { - return nil, err - } - if err := src.FillPath(cue.MakePath(), e.input); err != nil { - return nil, err - } - flow := cueflow.New( - &cueflow.Config{}, - src.Cue(), - newTaskFunc(noOpRunner), - ) - for _, t := range flow.Tasks() { - if err := localdirs(compiler.Wrap(t.Value())); err != nil { - return nil, err - } - } - - return dirs, nil +func (e *Environment) Context() *plancontext.Context { + return e.state.Context } // Up missing values in environment configuration, and write them to state. @@ -139,7 +86,7 @@ func (e *Environment) Up(ctx context.Context, s solver.Solver) error { flow := cueflow.New( &cueflow.Config{}, e.src.Cue(), - newTaskFunc(newPipelineRunner(e.computed, s, e.state.Platform)), + NewTaskFunc(NewPipelineRunner(e.computed, s, e.state.Context)), ) if err := flow.Run(ctx); err != nil { return err @@ -163,7 +110,7 @@ func (e *Environment) Down(ctx context.Context, _ *DownOpts) error { type QueryOpts struct{} -func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { +func NewTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { return func(flowVal cue.Value) (cueflow.Runner, error) { v := compiler.Wrap(flowVal) if !isComponent(v) { @@ -174,11 +121,7 @@ func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { } } -func noOpRunner(t *cueflow.Task) error { - return nil -} - -func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform string) cueflow.RunnerFunc { +func NewPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancontext.Context) cueflow.RunnerFunc { return cueflow.RunnerFunc(func(t *cueflow.Task) error { ctx := t.Context() lg := log. @@ -200,23 +143,7 @@ func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform strin } v := compiler.Wrap(t.Value()) - var pipelinePlatform specs.Platform - if platform == "" { - pipelinePlatform = specs.Platform{OS: "linux", Architecture: "amd64"} - } else { - p, err := platforms.Parse(platform) - if err != nil { - // Record the error - span.AddEvent("command", trace.WithAttributes( - attribute.String("error", err.Error()), - )) - - return err - } - pipelinePlatform = p - } - - p := NewPipeline(v, s, pipelinePlatform) + p := NewPipeline(v, s, pctx) err := p.Run(ctx) if err != nil { // Record the error diff --git a/environment/pipeline.go b/environment/pipeline.go index d0dfad72..8812444f 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -23,11 +23,11 @@ import ( bkgw "github.com/moby/buildkit/frontend/gateway/client" bkpb "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" ) @@ -46,17 +46,17 @@ type Pipeline struct { name string s solver.Solver state llb.State - platform specs.Platform // Platform constraint + pctx *plancontext.Context result bkgw.Reference image dockerfile2llb.Image computed *compiler.Value } -func NewPipeline(code *compiler.Value, s solver.Solver, platform specs.Platform) *Pipeline { +func NewPipeline(code *compiler.Value, s solver.Solver, pctx *plancontext.Context) *Pipeline { return &Pipeline{ code: code, name: code.Path().String(), - platform: platform, + pctx: pctx, s: s, state: llb.Scratch(), computed: compiler.NewValue(), @@ -233,7 +233,7 @@ func (p *Pipeline) run(ctx context.Context) error { // so that errors map to the correct cue path. // FIXME: might as well change FS to make every operation // synchronous. - p.result, err = p.s.Solve(ctx, p.state, p.platform) + p.result, err = p.s.Solve(ctx, p.state, p.pctx.Platform.Get()) if err != nil { return err } @@ -339,7 +339,7 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) ( return st, err } // Execute 'from' in a tmp pipeline, and use the resulting fs - from := NewPipeline(op.Lookup("from"), p.s, p.platform) + from := NewPipeline(op.Lookup("from"), p.s, p.pctx) if err := from.Run(ctx); err != nil { return st, err } @@ -361,50 +361,31 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) ( } func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { - dir, err := op.Lookup("dir").String() + id, err := op.Lookup("id").String() if err != nil { return st, err } + dir := p.pctx.Directories.Get(plancontext.ContextKey(id)) + if dir == nil { + return st, fmt.Errorf("directory %q not found", id) + } opts := []llb.LocalOption{ - llb.WithCustomName(p.vertexNamef("Local %s", dir)), + llb.WithCustomName(p.vertexNamef("Local %s", dir.Path)), // Without hint, multiple `llb.Local` operations on the // same path get a different digest. llb.SessionID(p.s.SessionID()), - llb.SharedKeyHint(dir), + llb.SharedKeyHint(dir.Path), } - includes, err := op.Lookup("include").List() - if err != nil { - return st, err - } - if len(includes) > 0 { - includePatterns := []string{} - for _, i := range includes { - pattern, err := i.String() - if err != nil { - return st, err - } - includePatterns = append(includePatterns, pattern) - } - opts = append(opts, llb.IncludePatterns(includePatterns)) - } - - excludes, err := op.Lookup("exclude").List() - if err != nil { - return st, err + if len(dir.Include) > 0 { + opts = append(opts, llb.IncludePatterns(dir.Include)) } // Excludes .dagger directory by default excludePatterns := []string{"**/.dagger/"} - if len(excludes) > 0 { - for _, i := range excludes { - pattern, err := i.String() - if err != nil { - return st, err - } - excludePatterns = append(excludePatterns, pattern) - } + if len(dir.Exclude) > 0 { + excludePatterns = dir.Exclude } opts = append(opts, llb.ExcludePatterns(excludePatterns)) @@ -415,13 +396,13 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) return st.File( llb.Copy( llb.Local( - dir, + dir.Path, opts..., ), "/", "/", ), - llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)), + llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir.Path)), ), nil } @@ -574,35 +555,15 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value) return nil, fmt.Errorf("invalid stream %q: not a stream", stream.Path().String()) } - // Unix socket - unixValue := stream.Lookup("unix") - if unixValue.Exists() { - unix, err := unixValue.String() - if err != nil { - return nil, fmt.Errorf("invalid unix path id: %w", err) - } - - return llb.AddSSHSocket( - llb.SSHID(fmt.Sprintf("unix=%s", unix)), - llb.SSHSocketTarget(dest), - ), nil + id, err := stream.Lookup("id").String() + if err != nil { + return nil, fmt.Errorf("invalid stream %q: %w", stream.Path().String(), err) } - // Windows named pipe - npipeValue := stream.Lookup("npipe") - if npipeValue.Exists() { - npipe, err := npipeValue.String() - if err != nil { - return nil, fmt.Errorf("invalid npipe path id: %w", err) - } - - return llb.AddSSHSocket( - llb.SSHID(fmt.Sprintf("npipe=%s", npipe)), - llb.SSHSocketTarget(dest), - ), nil - } - - return nil, fmt.Errorf("invalid stream %q: not a valid stream", stream.Path().String()) + return llb.AddSSHSocket( + llb.SSHID(id), + llb.SSHSocketTarget(dest), + ), nil } // eg. mount: "/foo": { from: www.source } @@ -610,7 +571,7 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value) return nil, fmt.Errorf("invalid mount: should have %s structure", "{from: _, path: string | *\"/\"}") } - from := NewPipeline(mnt.Lookup("from"), p.s, p.platform) + from := NewPipeline(mnt.Lookup("from"), p.s, p.pctx) if err := from.Run(ctx); err != nil { return nil, err } @@ -736,7 +697,7 @@ func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) { } // parseStringOrSecret retrieve secret as plain text or retrieve string -func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compiler.Value) (string, error) { +func parseStringOrSecret(pctx *plancontext.Context, v *compiler.Value) (string, error) { // Check if the value is a string, return as is if value, err := v.String(); err == nil { return value, nil @@ -747,16 +708,16 @@ func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compile if err != nil { return "", err } - secretBytes, err := ss.GetSecret(ctx, id) - if err != nil { - return "", err + secret := pctx.Secrets.Get(plancontext.ContextKey(id)) + if secret == nil { + return "", fmt.Errorf("secret %s not found", id) } - return string(secretBytes), nil + return secret.PlainText, nil } func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { // Execute 'from' in a tmp pipeline, and use the resulting fs - from := NewPipeline(op.Lookup("from"), p.s, p.platform) + from := NewPipeline(op.Lookup("from"), p.s, p.pctx) if err := from.Run(ctx); err != nil { return st, err } @@ -774,7 +735,7 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S // that function // But currently it's not possible because ECR secret's is a string // so we need to handle both options (string & secret) - secretValue, err := parseStringOrSecret(ctx, p.s.GetOptions().SecretsStore, op.Lookup("secret")) + secretValue, err := parseStringOrSecret(p.pctx, op.Lookup("secret")) if err != nil { return st, err } @@ -813,9 +774,10 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st ll ) // Load image metadata and convert to to LLB. + platform := p.pctx.Platform.Get() p.image, err = p.s.ResolveImageConfig(ctx, ref.String(), llb.ResolveImageConfigOpt{ LogName: p.vertexNamef("load metadata for %s", ref.String()), - Platform: &p.platform, + Platform: &platform, }) if err != nil { return st, err @@ -869,7 +831,7 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb "name": ref.String(), "push": "true", }, - }, p.platform) + }, p.pctx.Platform.Get()) if err != nil { return st, err @@ -928,7 +890,7 @@ func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.Sta Output: func(_ map[string]string) (io.WriteCloser, error) { return pipeW, nil }, - }, p.platform) + }, p.pctx.Platform.Get()) if err != nil { return st, err @@ -1088,7 +1050,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S // docker build context. This can come from another component, so we need to // compute it first. if dockerContext.Exists() { - from := NewPipeline(op.Lookup("context"), p.s, p.platform) + from := NewPipeline(op.Lookup("context"), p.s, p.pctx) if err := from.Run(ctx); err != nil { return st, err } @@ -1118,7 +1080,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S } } - opts, err := dockerBuildOpts(ctx, op, p.s.GetOptions().SecretsStore) + opts, err := dockerBuildOpts(op, p.pctx) if err != nil { return st, err } @@ -1129,7 +1091,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S // Set platform to configured one if no one is defined if opts["platform"] == "" { - opts["platform"] = bkplatforms.Format(p.platform) + opts["platform"] = bkplatforms.Format(p.pctx.Platform.Get()) } req := bkgw.SolveRequest{ @@ -1162,7 +1124,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S return applyImageToState(p.image, st), nil } -func dockerBuildOpts(ctx context.Context, op *compiler.Value, ss solver.SecretsStore) (map[string]string, error) { +func dockerBuildOpts(op *compiler.Value, pctx *plancontext.Context) (map[string]string, error) { opts := map[string]string{} if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() { @@ -1205,7 +1167,7 @@ func dockerBuildOpts(ctx context.Context, op *compiler.Value, ss solver.SecretsS return nil, err } for _, buildArg := range fields { - v, err := parseStringOrSecret(ctx, ss, buildArg.Value) + v, err := parseStringOrSecret(pctx, buildArg.Value) if err != nil { return nil, err } diff --git a/plancontext/context.go b/plancontext/context.go new file mode 100644 index 00000000..743756d0 --- /dev/null +++ b/plancontext/context.go @@ -0,0 +1,48 @@ +package plancontext + +import ( + "crypto/sha256" + "encoding/json" + "fmt" +) + +type ContextKey string + +// Context holds the execution context for a plan. +// +// Usage: +// ctx := plancontext.New() +// id := ctx.Secrets.Register("mysecret") +// secret := ctx.Secrets.Get(id) +type Context struct { + Platform *platformContext + Directories *directoryContext + Secrets *secretContext + Services *serviceContext +} + +func New() *Context { + return &Context{ + Platform: &platformContext{ + platform: defaultPlatform, + }, + Directories: &directoryContext{ + store: make(map[ContextKey]*Directory), + }, + Secrets: &secretContext{ + store: make(map[ContextKey]*Secret), + }, + Services: &serviceContext{ + store: make(map[ContextKey]*Service), + }, + } +} + +func hashID(v interface{}) ContextKey { + data, err := json.Marshal(v) + if err != nil { + panic(err) + } + hash := sha256.Sum256(data) + return ContextKey(fmt.Sprintf("%x", hash)) +} diff --git a/plancontext/directory.go b/plancontext/directory.go new file mode 100644 index 00000000..1bed1770 --- /dev/null +++ b/plancontext/directory.go @@ -0,0 +1,42 @@ +package plancontext + +import "sync" + +type Directory struct { + Path string + Include []string + Exclude []string +} + +type directoryContext struct { + l sync.RWMutex + store map[ContextKey]*Directory +} + +func (c *directoryContext) Register(directory *Directory) ContextKey { + c.l.Lock() + defer c.l.Unlock() + + id := hashID(directory) + c.store[id] = directory + return id +} + +func (c *directoryContext) Get(id ContextKey) *Directory { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[id] +} + +func (c *directoryContext) List() []*Directory { + c.l.RLock() + defer c.l.RUnlock() + + directories := make([]*Directory, 0, len(c.store)) + for _, d := range c.store { + directories = append(directories, d) + } + + return directories +} diff --git a/plancontext/platform.go b/plancontext/platform.go new file mode 100644 index 00000000..5d12aeab --- /dev/null +++ b/plancontext/platform.go @@ -0,0 +1,32 @@ +package plancontext + +import ( + "github.com/containerd/containerd/platforms" + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // Default platform. + // FIXME: This should be auto detected using buildkit + defaultPlatform = specs.Platform{ + OS: "linux", + Architecture: "amd64", + } +) + +type platformContext struct { + platform specs.Platform +} + +func (c *platformContext) Get() specs.Platform { + return c.platform +} + +func (c *platformContext) Set(platform string) error { + p, err := platforms.Parse(platform) + if err != nil { + return err + } + c.platform = p + return nil +} diff --git a/plancontext/secret.go b/plancontext/secret.go new file mode 100644 index 00000000..2c052d69 --- /dev/null +++ b/plancontext/secret.go @@ -0,0 +1,40 @@ +package plancontext + +import "sync" + +type secretContext struct { + l sync.RWMutex + store map[ContextKey]*Secret +} + +type Secret struct { + PlainText string +} + +func (c *secretContext) Register(secret *Secret) ContextKey { + c.l.Lock() + defer c.l.Unlock() + + id := hashID(secret.PlainText) + c.store[id] = secret + return id +} + +func (c *secretContext) Get(id ContextKey) *Secret { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[id] +} + +func (c *secretContext) List() []*Secret { + c.l.RLock() + defer c.l.RUnlock() + + secrets := make([]*Secret, 0, len(c.store)) + for _, s := range c.store { + secrets = append(secrets, s) + } + + return secrets +} diff --git a/plancontext/service.go b/plancontext/service.go new file mode 100644 index 00000000..3dbffdc2 --- /dev/null +++ b/plancontext/service.go @@ -0,0 +1,29 @@ +package plancontext + +import "sync" + +type serviceContext struct { + l sync.RWMutex + store map[ContextKey]*Service +} + +type Service struct { + Unix string + Npipe string +} + +func (c *serviceContext) Register(service *Service) ContextKey { + c.l.Lock() + defer c.l.Unlock() + + id := hashID(service) + c.store[id] = service + return id +} + +func (c *serviceContext) Get(id ContextKey) *Service { + c.l.RLock() + defer c.l.RUnlock() + + return c.store[id] +} diff --git a/solver/secretsprovider.go b/solver/secretsprovider.go index 95382e63..8e811bde 100644 --- a/solver/secretsprovider.go +++ b/solver/secretsprovider.go @@ -2,55 +2,27 @@ package solver import ( "context" - "strings" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/secrets" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/rs/zerolog/log" - "go.dagger.io/dagger/state" + "go.dagger.io/dagger/plancontext" ) -type SecretsStore struct { - Secrets session.Attachable - store *inputStore -} - -func (s SecretsStore) GetSecret(ctx context.Context, id string) ([]byte, error) { - return s.store.GetSecret(ctx, id) -} - -func NewSecretsStoreProvider(st *state.State) SecretsStore { - store := &inputStore{st} - - return SecretsStore{ - Secrets: secretsprovider.NewSecretProvider(store), - store: store, - } +func NewSecretsStoreProvider(pctx *plancontext.Context) session.Attachable { + return secretsprovider.NewSecretProvider(&inputStore{pctx}) } type inputStore struct { - st *state.State + pctx *plancontext.Context } func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) { lg := log.Ctx(ctx) - const secretPrefix = "secret=" - - if !strings.HasPrefix(id, secretPrefix) { - return nil, secrets.ErrNotFound - } - - id = strings.TrimPrefix(id, secretPrefix) - - id = strings.Split(id, ";hash=")[0] - - input, ok := s.st.Inputs[id] - if !ok { - return nil, secrets.ErrNotFound - } - if input.Secret == nil { + secret := s.pctx.Secrets.Get(plancontext.ContextKey(id)) + if secret == nil { return nil, secrets.ErrNotFound } @@ -59,5 +31,5 @@ func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) { Str("id", id). Msg("injecting secret") - return []byte(input.Secret.PlainText()), nil + return []byte(secret.PlainText), nil } diff --git a/solver/socketprovider.go b/solver/socketprovider.go index d0a0f52f..00d9ec94 100644 --- a/solver/socketprovider.go +++ b/solver/socketprovider.go @@ -3,24 +3,20 @@ package solver import ( "context" "fmt" - "strings" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/sshforward" + "go.dagger.io/dagger/plancontext" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) -const ( - unixPrefix = "unix=" - npipePrefix = "npipe=" -) - type SocketProvider struct { + pctx *plancontext.Context } -func NewDockerSocketProvider() session.Attachable { - return &SocketProvider{} +func NewDockerSocketProvider(pctx *plancontext.Context) session.Attachable { + return &SocketProvider{pctx} } func (sp *SocketProvider) Register(server *grpc.Server) { @@ -28,13 +24,6 @@ func (sp *SocketProvider) Register(server *grpc.Server) { } func (sp *SocketProvider) CheckAgent(ctx context.Context, req *sshforward.CheckAgentRequest) (*sshforward.CheckAgentResponse, error) { - id := sshforward.DefaultID - if req.ID != "" { - id = req.ID - } - if !strings.HasPrefix(id, unixPrefix) && !strings.HasPrefix(id, npipePrefix) { - return &sshforward.CheckAgentResponse{}, fmt.Errorf("invalid socket forward key %s", id) - } return &sshforward.CheckAgentResponse{}, nil } @@ -47,7 +36,12 @@ func (sp *SocketProvider) ForwardAgent(stream sshforward.SSH_ForwardAgentServer) id = v[0] } - conn, err := dialStream(id) + service := sp.pctx.Services.Get(plancontext.ContextKey(id)) + if service == nil { + return fmt.Errorf("invalid socket id %q", id) + } + + conn, err := dialService(service) if err != nil { return fmt.Errorf("failed to connect to %s: %w", id, err) } diff --git a/solver/socketprovider_unix.go b/solver/socketprovider_unix.go index 04e31fcc..529ed0ea 100644 --- a/solver/socketprovider_unix.go +++ b/solver/socketprovider_unix.go @@ -4,17 +4,17 @@ package solver import ( - "fmt" + "errors" "net" - "strings" "time" + + "go.dagger.io/dagger/plancontext" ) -func dialStream(id string) (net.Conn, error) { - if !strings.HasPrefix(id, unixPrefix) { - return nil, fmt.Errorf("invalid socket forward key %s", id) +func dialService(service *plancontext.Service) (net.Conn, error) { + if service.Unix == "" { + return nil, errors.New("unsupported socket type") } - id = strings.TrimPrefix(id, unixPrefix) - return net.DialTimeout("unix", id, time.Second) + return net.DialTimeout("unix", service.Unix, time.Second) } diff --git a/solver/socketprovider_windows.go b/solver/socketprovider_windows.go index aa47b625..0a425527 100644 --- a/solver/socketprovider_windows.go +++ b/solver/socketprovider_windows.go @@ -4,20 +4,19 @@ package solver import ( - "fmt" + "errors" "net" - "strings" "time" "github.com/Microsoft/go-winio" + "go.dagger.io/dagger/plancontext" ) -func dialStream(id string) (net.Conn, error) { - if !strings.HasPrefix(id, npipePrefix) { - return nil, fmt.Errorf("invalid socket forward key %s", id) +func dialService(service *plancontext.Service) (net.Conn, error) { + if service.Npipe == "" { + return nil, errors.New("unsupported socket type") } - id = strings.TrimPrefix(id, npipePrefix) dur := time.Second - return winio.DialPipe(id, &dur) + return winio.DialPipe(service.Npipe, &dur) } diff --git a/solver/solver.go b/solver/solver.go index 6e11fe4e..f5a57e61 100644 --- a/solver/solver.go +++ b/solver/solver.go @@ -18,6 +18,7 @@ import ( "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog/log" + "go.dagger.io/dagger/plancontext" ) type Solver struct { @@ -27,12 +28,12 @@ type Solver struct { } type Opts struct { - Control *bk.Client - Gateway bkgw.Client - Events chan *bk.SolveStatus - Auth *RegistryAuthProvider - SecretsStore SecretsStore - NoCache bool + Control *bk.Client + Gateway bkgw.Client + Events chan *bk.SolveStatus + Context *plancontext.Context + Auth *RegistryAuthProvider + NoCache bool } func New(opts Opts) Solver { @@ -194,8 +195,8 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im Exports: []bk.ExportEntry{output}, Session: []session.Attachable{ s.opts.Auth, - s.opts.SecretsStore.Secrets, - NewDockerSocketProvider(), + NewSecretsStoreProvider(s.opts.Context), + NewDockerSocketProvider(s.opts.Context), }, } diff --git a/state/input.go b/state/input.go index 22be56fc..6a2ccf42 100644 --- a/state/input.go +++ b/state/input.go @@ -1,8 +1,6 @@ package state import ( - "crypto/sha256" - "encoding/json" "fmt" "io/ioutil" "os" @@ -13,6 +11,7 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" ) // An input is a value or artifact supplied by the user. @@ -31,7 +30,6 @@ import ( type Input struct { Dir *dirInput `yaml:"dir,omitempty"` Git *gitInput `yaml:"git,omitempty"` - Docker *dockerInput `yaml:"docker,omitempty"` Secret *secretInput `yaml:"secret,omitempty"` Text *textInput `yaml:"text,omitempty"` JSON *jsonInput `yaml:"json,omitempty"` @@ -41,28 +39,26 @@ type Input struct { Socket *socketInput `yaml:"socket,omitempty"` } -func (i Input) Compile(key string, state *State) (*compiler.Value, error) { +func (i Input) Compile(state *State) (*compiler.Value, error) { switch { case i.Dir != nil: - return i.Dir.Compile(key, state) + return i.Dir.Compile(state) case i.Git != nil: - return i.Git.Compile(key, state) - case i.Docker != nil: - return i.Docker.Compile(key, state) + return i.Git.Compile(state) case i.Text != nil: - return i.Text.Compile(key, state) + return i.Text.Compile(state) case i.Secret != nil: - return i.Secret.Compile(key, state) + return i.Secret.Compile(state) case i.JSON != nil: - return i.JSON.Compile(key, state) + return i.JSON.Compile(state) case i.YAML != nil: - return i.YAML.Compile(key, state) + return i.YAML.Compile(state) case i.File != nil: - return i.File.Compile(key, state) + return i.File.Compile(state) case i.Bool != nil: - return i.Bool.Compile(key, state) + return i.Bool.Compile(state) case i.Socket != nil: - return i.Socket.Compile(key, state) + return i.Socket.Compile(state) default: return nil, fmt.Errorf("input has not been set") } @@ -85,28 +81,7 @@ type dirInput struct { Exclude []string `yaml:"exclude,omitempty"` } -func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) { - // FIXME: serialize an intermediate struct, instead of generating cue source - - // json.Marshal([]string{}) returns []byte("null"), which wreaks havoc - // in Cue because `null` is not a `[...string]` - includeLLB := []byte("[]") - if len(dir.Include) > 0 { - var err error - includeLLB, err = json.Marshal(dir.Include) - if err != nil { - return nil, err - } - } - excludeLLB := []byte("[]") - if len(dir.Exclude) > 0 { - var err error - excludeLLB, err = json.Marshal(dir.Exclude) - if err != nil { - return nil, err - } - } - +func (dir dirInput) Compile(state *State) (*compiler.Value, error) { p := dir.Path if !filepath.IsAbs(p) { p = filepath.Clean(path.Join(state.Project, dir.Path)) @@ -119,16 +94,15 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) { return nil, fmt.Errorf("%q dir doesn't exist", dir.Path) } - dirPath, err := json.Marshal(p) - if err != nil { - return nil, err - } + id := state.Context.Directories.Register(&plancontext.Directory{ + Path: p, + Include: dir.Include, + Exclude: dir.Exclude, + }) llb := fmt.Sprintf( - `#up: [{do:"local",dir:%s, include:%s, exclude:%s}]`, - dirPath, - includeLLB, - excludeLLB, + `#up: [{do:"local", id: "%s"}]`, + id, ) return compiler.Compile("", llb) } @@ -150,7 +124,7 @@ func GitInput(remote, ref, dir string) Input { } } -func (git gitInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (git gitInput) Compile(_ *State) (*compiler.Value, error) { ref := "HEAD" if git.Ref != "" { ref = git.Ref @@ -169,23 +143,6 @@ func (git gitInput) Compile(_ string, _ *State) (*compiler.Value, error) { )) } -// An input artifact loaded from a docker container -func DockerInput(ref string) Input { - return Input{ - Docker: &dockerInput{ - Ref: ref, - }, - } -} - -type dockerInput struct { - Ref string `yaml:"ref,omitempty"` -} - -func (i dockerInput) Compile(_ string, _ *State) (*compiler.Value, error) { - panic("NOT IMPLEMENTED") -} - // An input value encoded as text func TextInput(data string) Input { i := textInput(data) @@ -196,7 +153,7 @@ func TextInput(data string) Input { type textInput string -func (i textInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (i textInput) Compile(_ *State) (*compiler.Value, error) { return compiler.Compile("", fmt.Sprintf("%q", i)) } @@ -210,11 +167,11 @@ func SecretInput(data string) Input { type secretInput string -func (i secretInput) Compile(key string, _ *State) (*compiler.Value, error) { - hash := sha256.New() - hash.Write([]byte(key)) - checksum := hash.Sum([]byte(i.PlainText())) - secretValue := fmt.Sprintf(`{id:"secret=%s;hash=%x"}`, key, checksum) +func (i secretInput) Compile(st *State) (*compiler.Value, error) { + id := st.Context.Secrets.Register(&plancontext.Secret{ + PlainText: i.PlainText(), + }) + secretValue := fmt.Sprintf(`{id: %q}`, id) return compiler.Compile("", secretValue) } @@ -232,7 +189,7 @@ func BoolInput(data string) Input { type boolInput string -func (i boolInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (i boolInput) Compile(_ *State) (*compiler.Value, error) { s := map[boolInput]struct{}{ "true": {}, "false": {}, @@ -253,7 +210,7 @@ func JSONInput(data string) Input { type jsonInput string -func (i jsonInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (i jsonInput) Compile(_ *State) (*compiler.Value, error) { return compiler.DecodeJSON("", []byte(i)) } @@ -267,7 +224,7 @@ func YAMLInput(data string) Input { type yamlInput string -func (i yamlInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (i yamlInput) Compile(_ *State) (*compiler.Value, error) { return compiler.DecodeYAML("", []byte(i)) } @@ -283,7 +240,7 @@ type fileInput struct { Path string `yaml:"path,omitempty"` } -func (i fileInput) Compile(_ string, _ *State) (*compiler.Value, error) { +func (i fileInput) Compile(_ *State) (*compiler.Value, error) { data, err := ioutil.ReadFile(i.Path) if err != nil { return nil, err @@ -316,11 +273,11 @@ type socketInput struct { Npipe string `json:"npipe,omitempty" yaml:"npipe,omitempty"` } -func (i socketInput) Compile(_ string, _ *State) (*compiler.Value, error) { - socketValue, err := json.Marshal(i) - if err != nil { - return nil, err - } - - return compiler.Compile("", string(socketValue)) +func (i socketInput) Compile(st *State) (*compiler.Value, error) { + id := st.Context.Services.Register(&plancontext.Service{ + Unix: i.Unix, + Npipe: i.Npipe, + }) + socketValue := fmt.Sprintf(`{id: %q}`, id) + return compiler.Compile("", socketValue) } diff --git a/state/project.go b/state/project.go index 5dbd9c88..6c955fd2 100644 --- a/state/project.go +++ b/state/project.go @@ -12,6 +12,7 @@ import ( "github.com/rs/zerolog/log" "go.dagger.io/dagger/keychain" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/stdlib" "gopkg.in/yaml.v3" ) @@ -168,6 +169,12 @@ func (w *Project) Get(ctx context.Context, name string) (*State, error) { if err := yaml.Unmarshal(manifest, &st); err != nil { return nil, err } + st.Context = plancontext.New() + if platform := st.Platform; platform != "" { + if err := st.Context.Platform.Set(platform); err != nil { + return nil, err + } + } st.Path = envPath // FIXME: Backward compat: Support for old-style `.dagger/env//plan` if st.Plan.Module == "" { @@ -258,6 +265,8 @@ func (w *Project) Create(ctx context.Context, name string, plan Plan, platform s manifestPath := path.Join(envPath, manifestFile) st := &State{ + Context: plancontext.New(), + Path: envPath, Project: w.Path, Plan: Plan{ diff --git a/state/state.go b/state/state.go index a6251e59..b68a878e 100644 --- a/state/state.go +++ b/state/state.go @@ -6,10 +6,15 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" ) // Contents of an environment serialized to a file type State struct { + // Plan Context. + // FIXME: this is used as a bridge and is temporary. + Context *plancontext.Context `yaml:"-"` + // State path Path string `yaml:"-"` @@ -67,7 +72,7 @@ func (s *State) CompileInputs() (*compiler.Value, error) { // Prepare inputs for key, input := range s.Inputs { - i, err := input.Compile(key, s) + i, err := input.Compile(s) if err != nil { return nil, err } diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 0d236c6d..01a1944b 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -18,13 +18,7 @@ import ( #Stream: { @dagger(stream) - { - // Unix Socket - unix: string - } | { - // Windows Named Pipe - npipe: string - } + id: string } // Secret value diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index cf4d62c0..6587186c 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -20,10 +20,8 @@ package op } #Local: { - do: "local" - dir: string - include: [...string] - exclude: [...string] + do: "local" + id: string } // FIXME: bring back load (more efficient than copy) diff --git a/stdlib/docker/docker.cue b/stdlib/docker/docker.cue index 18413476..3fb7637c 100644 --- a/stdlib/docker/docker.cue +++ b/stdlib/docker/docker.cue @@ -160,6 +160,9 @@ import ( string #up: [ + // HACK: force a dependency with `load` + op.#Load & {from: load}, + op.#Load & {from: save}, op.#Export & { diff --git a/tests/core.bats b/tests/core.bats index 14866bb4..25fa80f2 100644 --- a/tests/core.bats +++ b/tests/core.bats @@ -145,7 +145,6 @@ setup() { # Make sure the secret doesn't show in dagger query run "$DAGGER" query mySecret.id -f text assert_success - assert_output --partial "secret=mySecret;hash=" } @test "core: stream" { From 22fb91548b9baf32ee267203f8f24b817140becc Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 17 Nov 2021 16:20:49 -0800 Subject: [PATCH 2/3] test: docker-run-ports: fix sandbox escaping of dagger stream Signed-off-by: Andrea Luzzardi --- stdlib/.dagger/env/docker-run-ports/values.yaml | 8 ++++++-- stdlib/docker/tests/run-ports/ports.cue | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/stdlib/.dagger/env/docker-run-ports/values.yaml b/stdlib/.dagger/env/docker-run-ports/values.yaml index ff47e285..500ee238 100644 --- a/stdlib/.dagger/env/docker-run-ports/values.yaml +++ b/stdlib/.dagger/env/docker-run-ports/values.yaml @@ -1,6 +1,10 @@ plan: package: ./docker/tests/run-ports name: docker-run-ports +inputs: + dockersocket: + socket: + unix: /var/run/docker.sock sops: kms: [] gcp_kms: [] @@ -16,8 +20,8 @@ sops: TS80V3BnN3BaeDFTRjNBRTAwbWRKcWcK1rH2zWxTQsbSgOs9Ys89ZCP7kVOm1wFe Kv849q+QOvIy9JPYkGgV16Yr3ijFF7z/h0PCicZRI76WYV/3JnbYmg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2021-09-28T15:06:47Z" - mac: ENC[AES256_GCM,data:WF2X4RCio9x459Fv4XK/P933cS9uac3kHA2+kfaft++ndxbHOY0zmOhUvNMmzLtJNTXL7oBADWIIqIiFbaaC0pC+xGq1th+woR3LvAVQi5J0rlDTcdMDPT03KOPdOmsXFDow5jzyn8vPsC59VCys0ZmEnmnHIGufnFy2qKVV0bM=,iv:GUrSk66zhDqxx+rVM9zMDfmcaDBSOOMIeCSViS2cuLU=,tag:rgCoWe//hn8xmeRmbc5ybQ==,type:str] + lastmodified: "2021-11-18T00:19:27Z" + mac: ENC[AES256_GCM,data:TtV35YtlXlZLvE8H+EoPvjJmCowmFftsS2DIifDeIeuRxkyMfk9tq39BemWFjoLKJx3FgFvC8bXGbWi3UMWi8wpzgjqmFVJCDWUdQ6NpdzPCXCVdOdkGUV+F/yjLX0dnsUZHnwSOZco0+uk/sNCWxecBE590brHoJw91j075DXQ=,iv:P8dEwdFnDvvMSJvNK4GXPMbTXbvLEsqoEhoP4K1IPVQ=,tag:LTT6gXDfSMWYttLOXQjidg==,type:str] pgp: [] encrypted_suffix: secret version: 3.7.1 diff --git a/stdlib/docker/tests/run-ports/ports.cue b/stdlib/docker/tests/run-ports/ports.cue index 95a395c0..e820f085 100644 --- a/stdlib/docker/tests/run-ports/ports.cue +++ b/stdlib/docker/tests/run-ports/ports.cue @@ -5,6 +5,8 @@ import ( "alpha.dagger.io/random" ) +dockersocket: dagger.#Stream & dagger.#Input + suffix: random.#String & { seed: "" } @@ -12,6 +14,6 @@ suffix: random.#String & { run: #Run & { name: "daggerci-test-ports-\(suffix.out)" ref: "nginx" - socket: dagger.#Stream & {unix: "/var/run/docker.sock"} + socket: dockersocket ports: ["8080:80"] } From 2beace385c40e331980389cf160607a484192107 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 18 Nov 2021 14:21:18 -0800 Subject: [PATCH 3/3] add test for plancontext Signed-off-by: Andrea Luzzardi --- plancontext/context_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 plancontext/context_test.go diff --git a/plancontext/context_test.go b/plancontext/context_test.go new file mode 100644 index 00000000..652c22fa --- /dev/null +++ b/plancontext/context_test.go @@ -0,0 +1,18 @@ +package plancontext + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestContext(t *testing.T) { + ctx := New() + + id := ctx.Secrets.Register(&Secret{ + PlainText: "test", + }) + secret := ctx.Secrets.Get(id) + require.NotNil(t, secret) + require.Equal(t, "test", secret.PlainText) +}