From c583dc20ef0968314ece0069260fc38f7aef038e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 31 Mar 2021 16:32:15 -0700 Subject: [PATCH 1/4] implemented ClientDoFunc callback to make the client.Up code reusable for other client actions Signed-off-by: Sam Alba --- dagger/client.go | 24 ++++++++++++++++-------- dagger/compiler/value.go | 4 ++-- dagger/deployment.go | 4 ---- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dagger/client.go b/dagger/client.go index 22030f5a..318beb93 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -61,8 +61,10 @@ func NewClient(ctx context.Context, host string) (*Client, error) { }, nil } +type ClientDoFunc func(context.Context, *Deployment, Solver) error + // FIXME: return completed *Route, instead of *compiler.Value -func (c *Client) Up(ctx context.Context, deployment *Deployment) (*compiler.Value, error) { +func (c *Client) Do(ctx context.Context, deployment *Deployment, fn ClientDoFunc) (*compiler.Value, error) { lg := log.Ctx(ctx) eg, gctx := errgroup.WithContext(ctx) @@ -79,7 +81,7 @@ func (c *Client) Up(ctx context.Context, deployment *Deployment) (*compiler.Valu outr, outw := io.Pipe() eg.Go(func() error { defer outw.Close() - return c.buildfn(gctx, deployment, events, outw) + return c.buildfn(gctx, deployment, fn, events, outw) }) // Spawn output retriever @@ -96,7 +98,7 @@ func (c *Client) Up(ctx context.Context, deployment *Deployment) (*compiler.Valu return out, eg.Wait() } -func (c *Client) buildfn(ctx context.Context, deployment *Deployment, ch chan *bk.SolveStatus, w io.WriteCloser) error { +func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientDoFunc, ch chan *bk.SolveStatus, w io.WriteCloser) error { lg := log.Ctx(ctx) // Scan local dirs to grant access @@ -138,9 +140,10 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, ch chan *b } // Compute output overlay - lg.Debug().Msg("computing deployment") - if err := deployment.Up(ctx, s, nil); err != nil { - return nil, compiler.Err(err) + if fn != nil { + if err := fn(ctx, deployment, s); err != nil { + return nil, compiler.Err(err) + } } // Export deployment to a cue directory @@ -149,9 +152,14 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, ch chan *b span, _ := opentracing.StartSpanFromContext(ctx, "Deployment.Export") defer span.Finish() + stateSource, err := deployment.State().Source() + if err != nil { + return nil, compiler.Err(err) + } + st := llb.Scratch().File( - llb.Mkfile("state.cue", 0600, deployment.State().JSON()), - llb.WithCustomName("[internal] serializing state to JSON"), + llb.Mkfile("state.cue", 0600, stateSource), + llb.WithCustomName("[internal] serializing state to CUE"), ) ref, err := s.Solve(ctx, st) if err != nil { diff --git a/dagger/compiler/value.go b/dagger/compiler/value.go index 2833134d..0dcaa1a6 100644 --- a/dagger/compiler/value.go +++ b/dagger/compiler/value.go @@ -204,11 +204,11 @@ func (v *Value) Validate() error { } // Return cue source for this value -func (v *Value) Source() ([]byte, error) { +func (v *Value) Source(opts ...cue.Option) ([]byte, error) { v.cc.rlock() defer v.cc.runlock() - return cueformat.Node(v.val.Eval().Syntax()) + return cueformat.Node(v.val.Eval().Syntax(opts...)) } func (v *Value) IsEmptyStruct() bool { diff --git a/dagger/deployment.go b/dagger/deployment.go index 8b0ad574..3726e49f 100644 --- a/dagger/deployment.go +++ b/dagger/deployment.go @@ -308,10 +308,6 @@ func (d *Deployment) Down(ctx context.Context, _ *DownOpts) error { panic("NOT IMPLEMENTED") } -func (d *Deployment) Query(ctx context.Context, expr interface{}, o *QueryOpts) (*compiler.Value, error) { - panic("NOT IMPLEMENTED") -} - type QueryOpts struct{} func newTaskFunc(inst *cue.Instance, runner cueflow.RunnerFunc) cueflow.TaskFunc { From ac5c8417d25237372a8aef85f90fefeb05da1765 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 31 Mar 2021 16:32:48 -0700 Subject: [PATCH 2/4] cmd: implemented query to be close to cue eval Signed-off-by: Sam Alba --- cmd/dagger/cmd/common/common.go | 7 ++- cmd/dagger/cmd/query.go | 85 +++++++++++++++++++++++++-------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 51cdfa59..2aeb7fe0 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -65,9 +65,12 @@ func DeploymentUp(ctx context.Context, deployment *dagger.Deployment) { if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } - output, err := c.Up(ctx, deployment) + output, err := c.Do(ctx, deployment, func(ctx context.Context, deployment *dagger.Deployment, s dagger.Solver) error { + log.Ctx(ctx).Debug().Msg("bringing deployment up") + return deployment.Up(ctx, s, nil) + }) if err != nil { - lg.Fatal().Err(err).Msg("failed to compute") + lg.Fatal().Err(err).Msg("failed to up deployment") } fmt.Println(output.JSON()) } diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index a0a7502c..ac267496 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -3,18 +3,20 @@ package cmd import ( "fmt" + "cuelang.org/go/cue" "dagger.io/go/cmd/dagger/cmd/common" "dagger.io/go/cmd/dagger/logger" "dagger.io/go/dagger" + "dagger.io/go/dagger/compiler" "github.com/spf13/cobra" "github.com/spf13/viper" ) var queryCmd = &cobra.Command{ - Use: "query [EXPR] [flags]", + Use: "query [TARGET] [flags]", Short: "Query the contents of a deployment", - Args: cobra.ExactArgs(1), + Args: cobra.MaximumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { // Fix Viper bug for duplicate flags: // https://github.com/spf13/viper/issues/233 @@ -26,6 +28,8 @@ var queryCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) + cueOpts := parseQueryFlags() + store, err := dagger.DefaultStore() if err != nil { lg.Fatal().Err(err).Msg("failed to load store") @@ -33,31 +37,74 @@ var queryCmd = &cobra.Command{ deployment := common.GetCurrentDeployment(ctx, store) - expr := args[0] + lg = lg.With(). + Str("deploymentName", deployment.Name()). + Str("deploymentId", deployment.ID()). + Logger() - out, err := deployment.Query(ctx, expr, nil) - if err != nil { - lg. - Fatal(). - Err(err). - Str("deploymentName", deployment.Name()). - Str("deploymentId", deployment.ID()). - Msg("failed to query deployment") + cuePath := cue.MakePath() + if len(args) > 0 { + cuePath = cue.ParsePath(args[0]) } - fmt.Println(out) - // TODO: Implement options: --no-*, --format, --revision + c, err := dagger.NewClient(ctx, "") + if err != nil { + lg.Fatal().Err(err).Msg("unable to create client") + } + output, err := c.Do(ctx, deployment, nil) + if err != nil { + lg.Fatal().Err(err).Msg("failed to query deployment") + } + cueVal := output.LookupPath(cuePath) + + if viper.GetBool("concrete") { + if err := cueVal.IsConcreteR(); err != nil { + lg.Fatal().Err(compiler.Err(err)).Msg("not concrete") + } + } + + out, err := cueVal.Source(cueOpts...) + if err != nil { + lg.Fatal().Err(err).Msg("failed to lookup source") + } + + output.IsConcreteR() + fmt.Println(string(out)) }, } -func init() { - queryCmd.Flags().String("revision", "latest", "Query a specific version of the deployment") - queryCmd.Flags().StringP("format", "f", "", "Output format (json|yaml|cue|text|env)") +func parseQueryFlags() []cue.Option { + opts := []cue.Option{ + cue.Definitions(true), + } - queryCmd.Flags().BoolP("--no-input", "I", false, "Exclude inputs from query") - queryCmd.Flags().BoolP("--no-output", "O", false, "Exclude outputs from query") - queryCmd.Flags().BoolP("--no-plan", "P", false, "Exclude outputs from query") + if viper.GetBool("concrete") { + opts = append(opts, cue.Concrete(true)) + } + + if viper.GetBool("show-optional") { + opts = append(opts, cue.Optional(true)) + } + + if viper.GetBool("show-attributes") { + opts = append(opts, cue.Attributes(true)) + } + + return opts +} + +func init() { + queryCmd.Flags().BoolP("concrete", "c", false, "Require the evaluation to be concrete") + queryCmd.Flags().BoolP("show-optional", "O", false, "Display optional fields") + queryCmd.Flags().BoolP("show-attributes", "A", false, "Display field attributes") + + // FIXME: implement the flags below + // queryCmd.Flags().String("revision", "latest", "Query a specific version of the deployment") + // queryCmd.Flags().StringP("format", "f", "", "Output format (json|yaml|cue|text|env)") + // queryCmd.Flags().BoolP("no-input", "I", false, "Exclude inputs from query") + // queryCmd.Flags().BoolP("no-output", "O", false, "Exclude outputs from query") + // queryCmd.Flags().BoolP("no-plan", "P", false, "Exclude outputs from query") if err := viper.BindPFlags(queryCmd.Flags()); err != nil { panic(err) From 3aec0337d27f6e8ecb6afc16c7543004b1c57bb8 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 31 Mar 2021 18:10:20 -0700 Subject: [PATCH 3/4] tests: fixed invalid test Signed-off-by: Sam Alba --- cmd/dagger/cmd/query.go | 1 - tests/push-container/main.cue | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index ac267496..1ff66ea2 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -69,7 +69,6 @@ var queryCmd = &cobra.Command{ lg.Fatal().Err(err).Msg("failed to lookup source") } - output.IsConcreteR() fmt.Println(string(out)) }, } diff --git a/tests/push-container/main.cue b/tests/push-container/main.cue index cb54bdf4..04676aad 100644 --- a/tests/push-container/main.cue +++ b/tests/push-container/main.cue @@ -1,7 +1,7 @@ package main import ( - "dagger.io/dagger" + "dagger.io/llb" "dagger.io/alpine" ) @@ -10,11 +10,11 @@ TestPushContainer: { random: { string #compute: [ - dagger.#Load & {from: alpine.#Image}, - dagger.#Exec & { + llb.#Load & {from: alpine.#Image}, + llb.#Exec & { args: ["sh", "-c", "echo -n $RANDOM > /rand"] }, - dagger.#Export & { + llb.#Export & { source: "/rand" }, ] @@ -24,11 +24,11 @@ TestPushContainer: { push: { ref: "daggerio/ci-test:\(random)" #compute: [ - dagger.#WriteFile & { + llb.#WriteFile & { content: random dest: "/rand" }, - dagger.#PushContainer & { + llb.#PushContainer & { "ref": ref }, ] @@ -36,15 +36,15 @@ TestPushContainer: { // Pull the image back pull: #compute: [ - dagger.#FetchContainer & { + llb.#FetchContainer & { ref: push.ref }, ] // Check the content check: #compute: [ - dagger.#Load & {from: alpine.#Image}, - dagger.#Exec & { + llb.#Load & {from: alpine.#Image}, + llb.#Exec & { args: [ "sh", "-c", #""" test "$(cat /src/rand)" = "\#(random)" From f1bf58ed86cedbff3d8a60003a49ed4cc06d17d6 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 31 Mar 2021 18:34:23 -0700 Subject: [PATCH 4/4] tests: fix invalid dockerbuild test Signed-off-by: Andrea Luzzardi --- tests/dockerbuild/main.cue | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/dockerbuild/main.cue b/tests/dockerbuild/main.cue index 6bb729e9..6c33dd32 100644 --- a/tests/dockerbuild/main.cue +++ b/tests/dockerbuild/main.cue @@ -1,12 +1,15 @@ package test -import "dagger.io/dagger" +import ( + "dagger.io/dagger" + "dagger.io/llb" +) // Set to `--input-dir=./tests/dockerbuild/testdata` TestData: dagger.#Artifact TestInlinedDockerfile: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d RUN echo hello world @@ -15,51 +18,51 @@ TestInlinedDockerfile: #compute: [ ] TestOpChaining: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d RUN echo foobar > /output """ }, - dagger.#Exec & { + llb.#Exec & { args: ["sh", "-c", "test $(cat /output) = foobar"] }, ] TestBuildContext: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { context: TestData }, - dagger.#Exec & { + llb.#Exec & { args: ["sh", "-c", "test $(cat /dir/foo) = foobar"] }, ] TestBuildContextAndDockerfile: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { context: TestData dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d COPY foo /override """ }, - dagger.#Exec & { + llb.#Exec & { args: ["sh", "-c", "test $(cat /override) = foobar"] }, ] TestDockerfilePath: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { context: TestData dockerfilePath: "./dockerfilepath/Dockerfile.custom" }, - dagger.#Exec & { + llb.#Exec & { args: ["sh", "-c", "test $(cat /test) = dockerfilePath"] }, ] TestBuildArgs: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d ARG TEST=foo @@ -71,7 +74,7 @@ TestBuildArgs: #compute: [ // FIXME: this doesn't test anything beside not crashing TestBuildLabels: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d """ @@ -81,7 +84,7 @@ TestBuildLabels: #compute: [ // FIXME: this doesn't test anything beside not crashing TestBuildPlatform: #compute: [ - dagger.#DockerBuild & { + llb.#DockerBuild & { dockerfile: """ FROM alpine:latest@sha256:ab00606a42621fb68f2ed6ad3c88be54397f981a7b70a79db3d1172b11c4367d """