From ce7adc1fa470b709ad8b6decb4c2f54ea27b1764 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 13 Jul 2021 14:55:49 +0200 Subject: [PATCH 1/4] cmd/up/compute: unify buildkit sessions to use only one Signed-off-by: Sam Alba --- cmd/dagger/cmd/common/common.go | 16 ------------- cmd/dagger/cmd/compute.go | 17 ++++++++++---- cmd/dagger/cmd/up.go | 40 +++++++++++++++------------------ 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 8285573f..7969e1e9 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -9,8 +9,6 @@ import ( "github.com/spf13/viper" "go.dagger.io/dagger/client" "go.dagger.io/dagger/compiler" - "go.dagger.io/dagger/environment" - "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" ) @@ -83,20 +81,6 @@ func CurrentEnvironmentState(ctx context.Context, workspace *state.Workspace) *s return environments[0] } -// Re-compute an environment (equivalent to `dagger up`). -func EnvironmentUp(ctx context.Context, cl *client.Client, state *state.State, noCache bool) *environment.Environment { - lg := log.Ctx(ctx) - - result, err := cl.Do(ctx, state, func(ctx context.Context, environment *environment.Environment, s solver.Solver) error { - log.Ctx(ctx).Debug().Msg("bringing environment up") - return environment.Up(ctx, s) - }) - if err != nil { - lg.Fatal().Err(err).Msg("failed to up environment") - } - return result -} - // FormatValue returns the String representation of the cue value func FormatValue(val *compiler.Value) string { if val.HasAttr("artifact") { diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index cdc2e2da..ebda5a8e 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "encoding/json" "errors" "fmt" @@ -8,10 +9,11 @@ import ( "strings" "cuelang.org/go/cue" - "go.dagger.io/dagger/client" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/decrypt" @@ -164,11 +166,18 @@ var computeCmd = &cobra.Command{ } } - cl, err := client.New(ctx, "", false) + cl := common.NewClient(ctx, viper.GetBool("no-cache")) + + environment, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + // check that all inputs are set + checkInputs(ctx, env) + + return env.Up(ctx, s) + }) + if err != nil { - lg.Fatal().Err(err).Msg("unable to create client") + lg.Fatal().Err(err).Msg("failed to up environment") } - environment := common.EnvironmentUp(ctx, cl, st, viper.GetBool("no-cache")) v := compiler.NewValue() if err := v.FillPath(cue.MakePath(), environment.Plan()); err != nil { diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 361e9e0f..b81e1ecd 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -5,14 +5,12 @@ import ( "os" "cuelang.org/go/cue" - "go.dagger.io/dagger/client" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/cmd/output" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" "go.dagger.io/dagger/solver" - "go.dagger.io/dagger/state" "golang.org/x/term" "github.com/rs/zerolog/log" @@ -40,10 +38,16 @@ var upCmd = &cobra.Command{ cl := common.NewClient(ctx, viper.GetBool("no-cache")) - // check that all inputs are set - checkInputs(ctx, cl, st) + result, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + // check that all inputs are set + checkInputs(ctx, env) - result := common.EnvironmentUp(ctx, cl, st, viper.GetBool("no-cache")) + return env.Up(ctx, s) + }) + + if err != nil { + lg.Fatal().Err(err).Msg("failed to up environment") + } st.Computed = result.Computed().JSON().PrettyString() if err := workspace.Save(ctx, st); err != nil { @@ -54,28 +58,20 @@ var upCmd = &cobra.Command{ }, } -func checkInputs(ctx context.Context, cl *client.Client, st *state.State) { +func checkInputs(ctx context.Context, env *environment.Environment) { lg := log.Ctx(ctx) warnOnly := viper.GetBool("force") notConcreteInputs := []*compiler.Value{} - _, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { - inputs, err := env.ScanInputs(ctx, true) - if err != nil { - return err - } - - for _, i := range inputs { - if i.IsConcreteR(cue.Optional(true)) != nil { - notConcreteInputs = append(notConcreteInputs, i) - } - } - - return nil - }) - + inputs, err := env.ScanInputs(ctx, true) if err != nil { - lg.Fatal().Err(err).Msg("failed to query environment") + lg.Fatal().Err(err).Msg("failed to scan inputs") + } + + for _, i := range inputs { + if i.IsConcreteR(cue.Optional(true)) != nil { + notConcreteInputs = append(notConcreteInputs, i) + } } for _, i := range notConcreteInputs { From ac32d6f57b68a49d8c20f5e8a3a23176ac0c3e69 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 13 Jul 2021 16:15:34 +0200 Subject: [PATCH 2/4] buildkit client does not return environment to force the caller to implement its logic in the build callback Signed-off-by: Sam Alba --- client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index dc4d1f1d..8267849e 100644 --- a/client/client.go +++ b/client/client.go @@ -69,13 +69,13 @@ func New(ctx context.Context, host string, noCache bool) (*Client, error) { type DoFunc func(context.Context, *environment.Environment, solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) (*environment.Environment, error) { +func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) environment, err := environment.New(state) if err != nil { - return nil, err + return err } // Spawn print function @@ -92,7 +92,7 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) (*enviro return c.buildfn(gctx, state, environment, fn, events) }) - return environment, eg.Wait() + return eg.Wait() } func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.Environment, fn DoFunc, ch chan *bk.SolveStatus) error { From 79d2e726a269651adbf9c4cb799aff08507e33f7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 13 Jul 2021 16:16:17 +0200 Subject: [PATCH 3/4] cmd: ported code to new client function signature, force one client per session Signed-off-by: Sam Alba --- cmd/dagger/cmd/compute.go | 33 +++++++++-------- cmd/dagger/cmd/edit.go | 2 +- cmd/dagger/cmd/input/list.go | 2 +- cmd/dagger/cmd/input/root.go | 2 +- cmd/dagger/cmd/output/list.go | 67 ++++++++++++++++------------------- cmd/dagger/cmd/query.go | 36 +++++++++---------- cmd/dagger/cmd/up.go | 33 ++++++++++------- 7 files changed, 90 insertions(+), 85 deletions(-) diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index ebda5a8e..2c758223 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -168,29 +168,32 @@ var computeCmd = &cobra.Command{ cl := common.NewClient(ctx, viper.GetBool("no-cache")) - environment, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { // check that all inputs are set checkInputs(ctx, env) - return env.Up(ctx, s) + if err := env.Up(ctx, s); err != nil { + return err + } + + v := compiler.NewValue() + if err := v.FillPath(cue.MakePath(), env.Plan()); err != nil { + return err + } + if err := v.FillPath(cue.MakePath(), env.Input()); err != nil { + return err + } + if err := v.FillPath(cue.MakePath(), env.Computed()); err != nil { + return err + } + + fmt.Println(v.JSON()) + return nil }) if err != nil { lg.Fatal().Err(err).Msg("failed to up environment") } - - v := compiler.NewValue() - if err := v.FillPath(cue.MakePath(), environment.Plan()); err != nil { - lg.Fatal().Err(err).Msg("failed to merge") - } - if err := v.FillPath(cue.MakePath(), environment.Input()); err != nil { - lg.Fatal().Err(err).Msg("failed to merge") - } - if err := v.FillPath(cue.MakePath(), environment.Computed()); err != nil { - lg.Fatal().Err(err).Msg("failed to merge") - } - - fmt.Println(v.JSON()) }, } diff --git a/cmd/dagger/cmd/edit.go b/cmd/dagger/cmd/edit.go index f4eae42b..60e5d5a7 100644 --- a/cmd/dagger/cmd/edit.go +++ b/cmd/dagger/cmd/edit.go @@ -71,7 +71,7 @@ var editCmd = &cobra.Command{ st.Inputs = newState.Inputs cl := common.NewClient(ctx, false) - _, err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { // check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index 2df921ed..8bd4d350 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -45,7 +45,7 @@ var listCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("unable to create client") } - _, err = c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err = c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { inputs, err := env.ScanInputs(ctx, false) if err != nil { return err diff --git a/cmd/dagger/cmd/input/root.go b/cmd/dagger/cmd/input/root.go index 8c32d613..f77ff73a 100644 --- a/cmd/dagger/cmd/input/root.go +++ b/cmd/dagger/cmd/input/root.go @@ -43,7 +43,7 @@ func updateEnvironmentInput(ctx context.Context, cl *client.Client, target strin st.SetInput(target, input) - _, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { // the inputs are set, check for cue errors by scanning all the inputs _, err := env.ScanInputs(ctx, true) if err != nil { diff --git a/cmd/dagger/cmd/output/list.go b/cmd/dagger/cmd/output/list.go index 983becc8..e1852069 100644 --- a/cmd/dagger/cmd/output/list.go +++ b/cmd/dagger/cmd/output/list.go @@ -6,12 +6,10 @@ import ( "os" "text/tabwriter" - "go.dagger.io/dagger/client" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/environment" "go.dagger.io/dagger/solver" - "go.dagger.io/dagger/state" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -36,51 +34,48 @@ var listCmd = &cobra.Command{ workspace := common.CurrentWorkspace(ctx) st := common.CurrentEnvironmentState(ctx, workspace) - ListOutputs(ctx, st, true) + cl := common.NewClient(ctx, false) + err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + return ListOutputs(ctx, env, true) + }) + + if err != nil { + lg.Fatal().Err(err).Msg("failed to scan outputs") + } }, } -func ListOutputs(ctx context.Context, st *state.State, isTTY bool) { +func ListOutputs(ctx context.Context, env *environment.Environment, isTTY bool) error { lg := log.Ctx(ctx).With(). - Str("environment", st.Name). + Str("environment", env.Name()). Logger() - c, err := client.New(ctx, "", false) + outputs, err := env.ScanOutputs(ctx) if err != nil { - lg.Fatal().Err(err).Msg("unable to create client") + lg.Error().Err(err).Msg("failed to scan outputs") + return err } - _, err = c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { - outputs, err := env.ScanOutputs(ctx) - if err != nil { - return err - } - - if !isTTY { - for _, out := range outputs { - lg.Info().Str("name", out.Path().String()). - Str("value", fmt.Sprintf("%v", out.Cue())). - Msg("output") - } - return nil - } - - w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) - fmt.Fprintln(w, "Output\tValue\tDescription") - + if !isTTY { for _, out := range outputs { - fmt.Fprintf(w, "%s\t%s\t%s\n", - out.Path(), - common.FormatValue(out), - common.ValueDocOneLine(out), - ) + lg.Info().Str("name", out.Path().String()). + Str("value", fmt.Sprintf("%v", out.Cue())). + Msg("output") } - - w.Flush() return nil - }) - - if err != nil { - lg.Fatal().Err(err).Msg("failed to query environment") } + + w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "Output\tValue\tDescription") + + for _, out := range outputs { + fmt.Fprintf(w, "%s\t%s\t%s\n", + out.Path(), + common.FormatValue(out), + common.ValueDocOneLine(out), + ) + } + + w.Flush() + return nil } diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index d4b895de..5d7390cf 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -1,13 +1,15 @@ package cmd import ( + "context" "fmt" "cuelang.org/go/cue" - "go.dagger.io/dagger/client" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/solver" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -42,28 +44,26 @@ var queryCmd = &cobra.Command{ cuePath = cue.ParsePath(args[0]) } - c, err := client.New(ctx, "", false) - if err != nil { - lg.Fatal().Err(err).Msg("unable to create client") - } - - environment, err := c.Do(ctx, state, nil) - if err != nil { - lg.Fatal().Err(err).Msg("failed to query environment") - } - + cl := common.NewClient(ctx, false) cueVal := compiler.NewValue() - if !viper.GetBool("no-plan") { - if err := cueVal.FillPath(cue.MakePath(), environment.Plan()); err != nil { - lg.Fatal().Err(err).Msg("failed to merge plan") + err := cl.Do(ctx, state, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + if !viper.GetBool("no-plan") { + if err := cueVal.FillPath(cue.MakePath(), env.Plan()); err != nil { + return err + } } - } - if !viper.GetBool("no-input") { - if err := cueVal.FillPath(cue.MakePath(), environment.Input()); err != nil { - lg.Fatal().Err(err).Msg("failed to merge plan with output") + if !viper.GetBool("no-input") { + if err := cueVal.FillPath(cue.MakePath(), env.Input()); err != nil { + return err + } } + return nil + }) + + if err != nil { + lg.Fatal().Err(err).Msg("failed to query environment") } if !viper.GetBool("no-computed") && state.Computed != "" { diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index b81e1ecd..c3f55597 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -38,34 +38,39 @@ var upCmd = &cobra.Command{ cl := common.NewClient(ctx, viper.GetBool("no-cache")) - result, err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { + err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error { // check that all inputs are set - checkInputs(ctx, env) + if err := checkInputs(ctx, env); err != nil { + return err + } - return env.Up(ctx, s) + if err := env.Up(ctx, s); err != nil { + return err + } + + st.Computed = env.Computed().JSON().PrettyString() + if err := workspace.Save(ctx, st); err != nil { + return err + } + + return output.ListOutputs(ctx, env, term.IsTerminal(int(os.Stdout.Fd()))) }) if err != nil { lg.Fatal().Err(err).Msg("failed to up environment") } - - st.Computed = result.Computed().JSON().PrettyString() - if err := workspace.Save(ctx, st); err != nil { - lg.Fatal().Err(err).Msg("failed to update environment") - } - - output.ListOutputs(ctx, st, term.IsTerminal(int(os.Stdout.Fd()))) }, } -func checkInputs(ctx context.Context, env *environment.Environment) { +func checkInputs(ctx context.Context, env *environment.Environment) error { lg := log.Ctx(ctx) warnOnly := viper.GetBool("force") notConcreteInputs := []*compiler.Value{} inputs, err := env.ScanInputs(ctx, true) if err != nil { - lg.Fatal().Err(err).Msg("failed to scan inputs") + lg.Error().Err(err).Msg("failed to scan inputs") + return err } for _, i := range inputs { @@ -83,8 +88,10 @@ func checkInputs(ctx context.Context, env *environment.Environment) { } if !warnOnly && len(notConcreteInputs) > 0 { - lg.Fatal().Int("missing", len(notConcreteInputs)).Msg("some required inputs are not set, please re-run with `--force` if you think it's a mistake") + lg.Error().Int("missing", len(notConcreteInputs)).Msg("some required inputs are not set, please re-run with `--force` if you think it's a mistake") } + + return nil } func init() { From f185088d8578f6ae08c4dd532a1f8f0e4acbb984 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 13 Jul 2021 16:37:37 +0200 Subject: [PATCH 4/4] cmd/up: fixed missing inputs error Signed-off-by: Sam Alba --- cmd/dagger/cmd/up.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index c3f55597..b438edd6 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "os" "cuelang.org/go/cue" @@ -88,7 +89,7 @@ func checkInputs(ctx context.Context, env *environment.Environment) error { } if !warnOnly && len(notConcreteInputs) > 0 { - lg.Error().Int("missing", len(notConcreteInputs)).Msg("some required inputs are not set, please re-run with `--force` if you think it's a mistake") + return errors.New("some required inputs are not set, please re-run with `--force` if you think it's a mistake") } return nil