diff --git a/client/client.go b/client/client.go index 8a6f189b..8569d9cd 100644 --- a/client/client.go +++ b/client/client.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/containerd/containerd/platforms" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "github.com/rs/zerolog/log" @@ -57,10 +58,9 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) { } opts := []bk.ClientOpt{} - // FIXME: uncomment when next version of buildkit will be released - // if span := trace.SpanFromContext(ctx); span != nil { - // opts = append(opts, bk.WithTracerProvider(span.TracerProvider())) - // } + if span := trace.SpanFromContext(ctx); span != nil { + opts = append(opts, bk.WithTracerProvider(span.TracerProvider())) + } c, err := bk.New(ctx, host, opts...) if err != nil { @@ -75,7 +75,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) { type DoFunc func(context.Context, solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error { +func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, localdirs map[string]string, fn DoFunc) error { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) @@ -90,13 +90,13 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) e // Spawn build function eg.Go(func() error { - return c.buildfn(gctx, pctx, fn, events) + return c.buildfn(gctx, pctx, localdirs, fn, events) }) return eg.Wait() } -func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error { +func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, localdirs map[string]string, fn DoFunc, ch chan *bk.SolveStatus) error { wg := sync.WaitGroup{} // Close output channel @@ -111,11 +111,6 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFu // buildkit auth provider (registry) auth := solver.NewRegistryAuthProvider() - localdirs := map[string]string{} - for _, dir := range pctx.Directories.List() { - localdirs[dir.Path] = dir.Path - } - // Setup solve options opts := bk.SolveOpt{ LocalDirs: localdirs, @@ -228,7 +223,7 @@ func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, lg := log. Ctx(ctx). With(). - Str("component", component). + Str("task", component). Logger() lg. @@ -243,7 +238,7 @@ func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, lg := log. Ctx(ctx). With(). - Str("component", component). + Str("task", component). Logger() msg := secureSprintf(format, a...) @@ -256,7 +251,7 @@ func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, lg := log. Ctx(ctx). With(). - Str("component", component). + Str("task", component). Logger() msg := secureSprintf(format, a...) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 9d4e1997..c9955894 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "cuelang.org/go/cue" "github.com/docker/buildx/util/buildflags" "github.com/rs/zerolog/log" "github.com/spf13/viper" @@ -91,9 +92,9 @@ func FormatValue(val *compiler.Value) string { return "dagger.#Secret" } if val.IsConcreteR() != nil { - return val.IncompleteKindString() + return val.IncompleteKind().String() } - if val.IncompleteKindString() == "struct" { + if val.IncompleteKind() == cue.StructKind { return "struct" } diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 320d87be..bb942a8d 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -198,7 +198,7 @@ var computeCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("unable to create environment") } - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set checkInputs(ctx, env) diff --git a/cmd/dagger/cmd/edit.go b/cmd/dagger/cmd/edit.go index a80a2343..6f9bfcf5 100644 --- a/cmd/dagger/cmd/edit.go +++ b/cmd/dagger/cmd/edit.go @@ -83,7 +83,7 @@ var editCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { // check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index e66aac28..712140f5 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -47,7 +47,7 @@ var listCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { inputs, err := env.ScanInputs(ctx, false) if err != nil { return err diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index 57b1a785..ff2f10ac 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -59,7 +59,7 @@ func updateEnvironmentInput(ctx context.Context, cmd *cobra.Command, target stri } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { // the inputs are set, check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/output/list.go b/cmd/dagger/cmd/output/list.go index 36887693..ffcb0dad 100644 --- a/cmd/dagger/cmd/output/list.go +++ b/cmd/dagger/cmd/output/list.go @@ -46,7 +46,7 @@ var listCmd = &cobra.Command{ } cl := common.NewClient(ctx) - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { return ListOutputs(ctx, env, true) }) diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index 1a01a811..e784927e 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -36,6 +36,8 @@ func init() { rootCmd.PersistentFlags().StringP("environment", "e", "", "Select an environment") rootCmd.PersistentFlags().String("project", "", "Specify a project directory (defaults to current)") + rootCmd.PersistentFlags().Bool("europa", false, "Enable experiemental Europa UX") + rootCmd.PersistentPreRun = func(cmd *cobra.Command, _ []string) { lg := logger.New() ctx := lg.WithContext(cmd.Context()) @@ -89,8 +91,7 @@ func Execute() { ) if len(os.Args) > 1 { - tr := otel.Tracer("cmd") - ctx, span = tr.Start(ctx, os.Args[1]) + ctx, span = otel.Tracer("dagger").Start(ctx, os.Args[1]) // Record the action span.AddEvent("command", trace.WithAttributes( attribute.String("args", strings.Join(os.Args, " ")), diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 947647cb..cffd0648 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -6,11 +6,13 @@ import ( "os" "cuelang.org/go/cue" + "go.dagger.io/dagger/client" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/cmd/output" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plan" "go.dagger.io/dagger/solver" "golang.org/x/term" @@ -61,12 +63,24 @@ var upCmd = &cobra.Command{ cl := common.NewClient(ctx) + if viper.GetBool("europa") { + err = europaUp(ctx, cl, project.Path) + + <-doneCh + + if err != nil { + lg.Fatal().Err(err).Msg("failed to up environment") + } + + return + } + env, err := environment.New(st) if err != nil { lg.Fatal().Err(err).Msg("unable to create environment") } - err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, env.Context(), env.Context().Directories.Paths(), func(ctx context.Context, s solver.Solver) error { // check that all inputs are set if err := checkInputs(ctx, env); err != nil { return err @@ -97,6 +111,27 @@ var upCmd = &cobra.Command{ }, } +func europaUp(ctx context.Context, cl *client.Client, path string) error { + lg := log.Ctx(ctx) + + p, err := plan.Load(ctx, path, "") + if err != nil { + lg.Fatal().Err(err).Msg("failed to load plan") + } + + localdirs, err := p.LocalDirectories() + if err != nil { + return err + } + return cl.Do(ctx, p.Context(), localdirs, func(ctx context.Context, s solver.Solver) error { + if err := p.Up(ctx, s); err != nil { + return err + } + + return nil + }) +} + func checkInputs(ctx context.Context, env *environment.Environment) error { lg := log.Ctx(ctx) warnOnly := viper.GetBool("force") diff --git a/cmd/dagger/logger/plain.go b/cmd/dagger/logger/plain.go index e454147e..b52b4b5e 100644 --- a/cmd/dagger/logger/plain.go +++ b/cmd/dagger/logger/plain.go @@ -118,7 +118,7 @@ func formatMessage(event map[string]interface{}) string { func parseSource(event map[string]interface{}) string { source := "system" - if task, ok := event["component"].(string); ok && task != "" { + if task, ok := event["task"].(string); ok && task != "" { source = task } return source @@ -141,7 +141,7 @@ func formatFields(entry map[string]interface{}) string { zerolog.ErrorFieldName: {}, zerolog.CallerFieldName: {}, "environment": {}, - "component": {}, + "task": {}, "state": {}, } diff --git a/cmd/dagger/logger/tty.go b/cmd/dagger/logger/tty.go index 7b7c650c..fa19172d 100644 --- a/cmd/dagger/logger/tty.go +++ b/cmd/dagger/logger/tty.go @@ -43,7 +43,7 @@ func (l *Logs) Add(event Event) error { l.l.Lock() defer l.l.Unlock() - component, ok := event["component"].(string) + task, ok := event["task"].(string) if !ok { l.Messages = append(l.Messages, Message{ Event: event, @@ -52,7 +52,7 @@ func (l *Logs) Add(event Event) error { return nil } - groupKey := strings.Split(component, ".#up")[0] + groupKey := strings.Split(task, ".#up")[0] group := l.groups[groupKey] // If the group doesn't exist, create it @@ -72,8 +72,8 @@ func (l *Logs) Add(event Event) error { // For state events, we just want to update the group status -- no need to // dispanything if st, ok := event["state"].(string); ok { - // Ignore state updates for "sub" components - if component != groupKey { + // Ignore state updates for "sub" tasks + if task != groupKey { return nil } diff --git a/compiler/value.go b/compiler/value.go index 167f3861..cfcddbc0 100644 --- a/compiler/value.go +++ b/compiler/value.go @@ -59,6 +59,11 @@ func (v *Value) Kind() cue.Kind { return v.val.Kind() } +// Proxy function to the underlying cue.Value +func (v *Value) IncompleteKind() cue.Kind { + return v.Cue().IncompleteKind() +} + // Field represents a struct field type Field struct { Selector cue.Selector @@ -145,6 +150,10 @@ func (v *Value) List() ([]*Value, error) { return l, nil } +func (v *Value) IsConcrete() bool { + return v.val.IsConcrete() +} + // Recursive concreteness check. func (v *Value) IsConcreteR(opts ...cue.Option) error { o := []cue.Option{cue.Concrete(true)} @@ -220,15 +229,6 @@ func (v *Value) Source(opts ...cue.Option) ([]byte, error) { ) } -func (v *Value) IsEmptyStruct() bool { - if st, err := v.Struct(); err == nil { - if st.Len() == 0 { - return true - } - } - return false -} - func (v *Value) Cue() cue.Value { return v.val } @@ -275,7 +275,3 @@ func (v *Value) Default() (*Value, bool) { func (v *Value) Doc() []*ast.CommentGroup { return v.Cue().Doc() } - -func (v *Value) IncompleteKindString() string { - return v.Cue().IncompleteKind().String() -} diff --git a/docs/reference/dagger/README.md b/docs/reference/dagger/README.md index 78a571c5..779f95f0 100644 --- a/docs/reference/dagger/README.md +++ b/docs/reference/dagger/README.md @@ -10,6 +10,28 @@ Dagger core types import "alpha.dagger.io/dagger" ``` +## dagger.#Context + +### dagger.#Context Inputs + +_No input._ + +### dagger.#Context Outputs + +_No output._ + +## dagger.#Plan + +A deployment plan executed by `dagger up` + +### dagger.#Plan Inputs + +_No input._ + +### dagger.#Plan Outputs + +_No output._ + ## dagger.#Secret Secret value diff --git a/environment/environment.go b/environment/environment.go index cd290b54..fe4b4609 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -78,15 +78,14 @@ func (e *Environment) Context() *plancontext.Context { // Up missing values in environment configuration, and write them to state. func (e *Environment) Up(ctx context.Context, s solver.Solver) error { - tr := otel.Tracer("environment") - ctx, span := tr.Start(ctx, "environment.Up") + ctx, span := otel.Tracer("dagger").Start(ctx, "environment.Up") defer span.End() // Orchestrate execution with cueflow flow := cueflow.New( &cueflow.Config{}, e.src.Cue(), - NewTaskFunc(NewPipelineRunner(e.computed, s, e.state.Context)), + newTaskFunc(newPipelineRunner(e.computed, s, e.state.Context)), ) if err := flow.Run(ctx); err != nil { return err @@ -110,10 +109,10 @@ func (e *Environment) Down(ctx context.Context, _ *DownOpts) error { type QueryOpts struct{} -func NewTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { +func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { return func(flowVal cue.Value) (cueflow.Runner, error) { v := compiler.Wrap(flowVal) - if !isComponent(v) { + if !IsComponent(v) { // No compute script return nil, nil } @@ -121,18 +120,17 @@ func NewTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc { } } -func NewPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancontext.Context) cueflow.RunnerFunc { +func newPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancontext.Context) cueflow.RunnerFunc { return cueflow.RunnerFunc(func(t *cueflow.Task) error { ctx := t.Context() lg := log. Ctx(ctx). With(). - Str("component", t.Path().String()). + Str("task", t.Path().String()). Logger() ctx = lg.WithContext(ctx) - tr := otel.Tracer("environment") - ctx, span := tr.Start(ctx, fmt.Sprintf("compute: %s", t.Path().String())) + ctx, span := otel.Tracer("dagger").Start(ctx, fmt.Sprintf("compute: %s", t.Path().String())) defer span.End() for _, dep := range t.Dependencies() { @@ -155,7 +153,7 @@ func NewPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancont } // Mirror the computed values in both `Task` and `Result` - if p.Computed().IsEmptyStruct() { + if !p.Computed().IsConcrete() { return nil } diff --git a/environment/pipeline.go b/environment/pipeline.go index 8812444f..34c839b1 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -91,14 +91,14 @@ func (p *Pipeline) Computed() *compiler.Value { return p.computed } -func isComponent(v *compiler.Value) bool { +func IsComponent(v *compiler.Value) bool { return v.Lookup("#up").Exists() } func ops(code *compiler.Value) ([]*compiler.Value, error) { ops := []*compiler.Value{} // 1. attachment array - if isComponent(code) { + if IsComponent(code) { xops, err := code.Lookup("#up").List() if err != nil { return nil, err @@ -166,7 +166,7 @@ func (p *Pipeline) Run(ctx context.Context) error { lg := log. Ctx(ctx). With(). - Str("component", p.name). + Str("task", p.name). Logger() start := time.Now() diff --git a/plan/plan.go b/plan/plan.go new file mode 100644 index 00000000..49f5a455 --- /dev/null +++ b/plan/plan.go @@ -0,0 +1,173 @@ +package plan + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "cuelang.org/go/cue" + cueflow "cuelang.org/go/tools/flow" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plan/task" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" + "go.dagger.io/dagger/state" + "go.opentelemetry.io/otel" +) + +type Plan struct { + context *plancontext.Context + source *compiler.Value +} + +func Load(ctx context.Context, path, pkg string) (*Plan, error) { + // FIXME: universe vendoring + if err := state.VendorUniverse(ctx, path); err != nil { + return nil, err + } + + v, err := compiler.Build(path, nil, pkg) + if err != nil { + return nil, err + } + + return &Plan{ + context: plancontext.New(), + source: v, + }, nil +} + +func (p *Plan) Context() *plancontext.Context { + return p.context +} + +func (p *Plan) Source() *compiler.Value { + return p.source +} + +// LocalDirectories scans the context for local imports. +// BuildKit requires to known the list of directories ahead of time. +func (p *Plan) LocalDirectories() (map[string]string, error) { + dirs := map[string]string{} + + imports, err := p.source.Lookup("context.imports").Fields() + if err != nil { + return nil, err + } + + for _, v := range imports { + dir, err := v.Value.Lookup("path").String() + if err != nil { + return nil, err + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + dirs[dir] = abs + } + + return dirs, nil +} + +// Up executes the plan +func (p *Plan) Up(ctx context.Context, s solver.Solver) error { + ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up") + defer span.End() + + computed := compiler.NewValue() + + flow := cueflow.New( + &cueflow.Config{}, + p.source.Cue(), + newRunner(p.context, s, computed), + ) + if err := flow.Run(ctx); err != nil { + return err + } + + if src, err := computed.Source(); err == nil { + log.Ctx(ctx).Debug().Str("computed", string(src)).Msg("computed values") + } + + // FIXME: canceling the context makes flow return `nil` + // Check explicitly if the context is canceled. + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func newRunner(pctx *plancontext.Context, s solver.Solver, computed *compiler.Value) cueflow.TaskFunc { + return func(flowVal cue.Value) (cueflow.Runner, error) { + v := compiler.Wrap(flowVal) + r, err := task.Lookup(v) + if err != nil { + // Not a task + if err == task.ErrNotTask { + return nil, nil + } + return nil, err + } + + // Wrapper around `task.Run` that handles logging, tracing, etc. + return cueflow.RunnerFunc(func(t *cueflow.Task) error { + ctx := t.Context() + lg := log.Ctx(ctx).With().Str("task", t.Path().String()).Logger() + ctx = lg.WithContext(ctx) + ctx, span := otel.Tracer("dagger").Start(ctx, fmt.Sprintf("compute: %s", t.Path().String())) + defer span.End() + + lg.Info().Str("state", string(environment.StateComputing)).Msg(string(environment.StateComputing)) + + // Debug: dump dependencies + for _, dep := range t.Dependencies() { + lg.Debug().Str("dependency", dep.Path().String()).Msg("dependency detected") + } + + start := time.Now() + result, err := r.Run(ctx, pctx, s, compiler.Wrap(t.Value())) + if err != nil { + // FIXME: this should use errdefs.IsCanceled(err) + if strings.Contains(err.Error(), "context canceled") { + lg.Error().Dur("duration", time.Since(start)).Str("state", string(environment.StateCanceled)).Msg(string(environment.StateCanceled)) + } else { + lg.Error().Dur("duration", time.Since(start)).Err(err).Str("state", string(environment.StateFailed)).Msg(string(environment.StateFailed)) + } + return err + } + + lg.Info().Dur("duration", time.Since(start)).Str("state", string(environment.StateCompleted)).Msg(string(environment.StateCompleted)) + + // If the result is not concrete, there's nothing to merge. + if !result.IsConcrete() { + return nil + } + + if src, err := result.Source(); err == nil { + lg.Debug().Str("result", string(src)).Msg("merging task result") + } + + // Mirror task result in both `flow.Task` and `computed` + if err := t.Fill(result.Cue()); err != nil { + lg.Error().Err(err).Msg("failed to fill task") + return err + } + + // Merge task value into computed + if err := computed.FillPath(t.Path(), result); err != nil { + lg.Error().Err(err).Msg("failed to fill plan") + return err + } + + return nil + }), nil + } +} diff --git a/plan/task/import.go b/plan/task/import.go new file mode 100644 index 00000000..40b4e2f9 --- /dev/null +++ b/plan/task/import.go @@ -0,0 +1,37 @@ +package task + +import ( + "context" + "fmt" + "os" + + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Import", func() Task { return &importTask{} }) +} + +type importTask struct { +} + +func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + var dir *plancontext.Directory + + if err := v.Decode(&dir); err != nil { + return nil, err + } + + // Check that directory exists + if _, err := os.Stat(dir.Path); os.IsNotExist(err) { + return nil, fmt.Errorf("%q dir doesn't exist", dir.Path) + } + + id := pctx.Directories.Register(dir) + return compiler.Compile("", fmt.Sprintf( + `fs: #up: [{do: "local", id: %q}]`, + id, + )) +} diff --git a/plan/task/pipeline.go b/plan/task/pipeline.go new file mode 100644 index 00000000..5cf02519 --- /dev/null +++ b/plan/task/pipeline.go @@ -0,0 +1,27 @@ +package task + +import ( + "context" + + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("#up", func() Task { return &pipelineTask{} }) +} + +// pipelineTask is an adapter for legacy pipelines (`#up`). +// FIXME: remove once fully migrated to Europa. +type pipelineTask struct { +} + +func (c pipelineTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + p := environment.NewPipeline(v, s, pctx) + if err := p.Run(ctx); err != nil { + return nil, err + } + return p.Computed(), nil +} diff --git a/plan/task/secretenv.go b/plan/task/secretenv.go new file mode 100644 index 00000000..11ee5d71 --- /dev/null +++ b/plan/task/secretenv.go @@ -0,0 +1,48 @@ +package task + +import ( + "context" + "fmt" + "os" + + "cuelang.org/go/cue" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("SecretEnv", func() Task { return &secretEnvTask{} }) +} + +type secretEnvTask struct { +} + +func (c secretEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + + var secretEnv struct { + Envvar string + } + + if err := v.Decode(&secretEnv); err != nil { + return nil, err + } + + lg.Debug().Str("envvar", secretEnv.Envvar).Msg("loading secret") + + env := os.Getenv(secretEnv.Envvar) + if env == "" { + return nil, fmt.Errorf("environment variable %q not set", secretEnv.Envvar) + } + id := pctx.Secrets.Register(&plancontext.Secret{ + PlainText: env, + }) + + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("contents.id"), id); err != nil { + return nil, err + } + return out, nil +} diff --git a/plan/task/secretfile.go b/plan/task/secretfile.go new file mode 100644 index 00000000..5d38cd53 --- /dev/null +++ b/plan/task/secretfile.go @@ -0,0 +1,47 @@ +package task + +import ( + "context" + "os" + + "cuelang.org/go/cue" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("SecretFile", func() Task { return &secretFileTask{} }) +} + +type secretFileTask struct { +} + +func (c secretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + + var secretFile struct { + Path string + } + + if err := v.Decode(&secretFile); err != nil { + return nil, err + } + + lg.Debug().Str("path", secretFile.Path).Msg("loading secret") + + data, err := os.ReadFile(secretFile.Path) + if err != nil { + return nil, err + } + id := pctx.Secrets.Register(&plancontext.Secret{ + PlainText: string(data), + }) + + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("contents.id"), id); err != nil { + return nil, err + } + return out, nil +} diff --git a/plan/task/task.go b/plan/task/task.go new file mode 100644 index 00000000..784c5b9a --- /dev/null +++ b/plan/task/task.go @@ -0,0 +1,68 @@ +package task + +import ( + "context" + "errors" + "fmt" + "sync" + + "cuelang.org/go/cue" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +var ( + ErrNotTask = errors.New("not a task") + tasks sync.Map + typePath = cue.MakePath(cue.Hid("_type", "alpha.dagger.io/dagger")) +) + +type NewFunc func() Task + +type Task interface { + Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) +} + +// Register a task type and initializer +func Register(typ string, f NewFunc) { + tasks.Store(typ, f) +} + +// New creates a new Task of the given type. +func New(typ string) Task { + v, ok := tasks.Load(typ) + if !ok { + return nil + } + fn := v.(NewFunc) + return fn() +} + +func Lookup(v *compiler.Value) (Task, error) { + // FIXME: legacy pipelines + if environment.IsComponent(v) { + return New("#up"), nil + } + + if v.Kind() != cue.StructKind { + return nil, ErrNotTask + } + + typ := v.LookupPath(typePath) + if !typ.Exists() { + return nil, ErrNotTask + } + + typeString, err := typ.String() + if err != nil { + return nil, err + } + + t := New(typeString) + if t == nil { + return nil, fmt.Errorf("unknown type %q", typeString) + } + return t, nil +} diff --git a/plancontext/directory.go b/plancontext/directory.go index 1bed1770..3011f682 100644 --- a/plancontext/directory.go +++ b/plancontext/directory.go @@ -40,3 +40,15 @@ func (c *directoryContext) List() []*Directory { return directories } + +func (c *directoryContext) Paths() map[string]string { + c.l.RLock() + defer c.l.RUnlock() + + directories := make(map[string]string) + for _, d := range c.store { + directories[d.Path] = d.Path + } + + return directories +} diff --git a/state/project.go b/state/project.go index 6c955fd2..db99b71d 100644 --- a/state/project.go +++ b/state/project.go @@ -55,7 +55,7 @@ func Init(ctx context.Context, dir string) (*Project, error) { return nil, err } - if err := vendorUniverse(ctx, root); err != nil { + if err := VendorUniverse(ctx, root); err != nil { return nil, err } @@ -390,7 +390,7 @@ func cueModInit(ctx context.Context, parentDir string) error { return nil } -func vendorUniverse(ctx context.Context, p string) error { +func VendorUniverse(ctx context.Context, p string) error { // ensure cue module is initialized if err := cueModInit(ctx, p); err != nil { return err diff --git a/state/state.go b/state/state.go index b68a878e..e68126e0 100644 --- a/state/state.go +++ b/state/state.go @@ -55,7 +55,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) { // 2) For backward compatibility: if the project was `dagger // init`-ed before we added support for vendoring universe, it might not // contain a `cue.mod`. - if err := vendorUniverse(ctx, w); err != nil { + if err := VendorUniverse(ctx, w); err != nil { return nil, err } diff --git a/stdlib/dagger/plan.cue b/stdlib/dagger/plan.cue new file mode 100644 index 00000000..ec71aa9b --- /dev/null +++ b/stdlib/dagger/plan.cue @@ -0,0 +1,41 @@ +package dagger + +// A deployment plan executed by `dagger up` +#Plan: { + context: #Context + actions: [string]: _ +} + +// FIXME: Platform spec here +#Platform: string + +#Context: { + // Platform to target + platform?: #Platform + + // Import directories + imports: [string]: { + _type: "Import" + + path: string + include?: [...string] + exclude?: [...string] + fs: #Artifact + } + + // Securely load external secrets + secrets: [string]: { + // Secrets can be securely mounted into action containers as a file + contents: #Secret + + { + _type: "SecretFile" + // Read secret from a file + path: string + } | { + _type: "SecretEnv" + // Read secret from an environment variable ON THE CLIENT MACHINE + envvar: string + } + } +}