From 83f56897a75dffd9128f58f66aad91fac7a5ca99 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 14 Apr 2021 16:21:18 -0700 Subject: [PATCH 1/6] solver/client: add ability to disable the cache at the LLB level Signed-off-by: Sam Alba --- dagger/client.go | 10 ++++++---- dagger/solver.go | 31 ++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/dagger/client.go b/dagger/client.go index 0d5cc719..3635301a 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -29,10 +29,11 @@ import ( // A dagger client type Client struct { - c *bk.Client + c *bk.Client + noCache bool } -func NewClient(ctx context.Context, host string) (*Client, error) { +func NewClient(ctx context.Context, host string, noCache bool) (*Client, error) { if host == "" { host = os.Getenv("BUILDKIT_HOST") } @@ -53,7 +54,8 @@ func NewClient(ctx context.Context, host string) (*Client, error) { return nil, fmt.Errorf("buildkit client: %w", err) } return &Client{ - c: c, + c: c, + noCache: noCache, }, nil } @@ -111,7 +113,7 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD Msg("spawning buildkit job") resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { - s := NewSolver(c.c, gw, ch) + s := NewSolver(c.c, gw, ch, c.noCache) lg.Debug().Msg("loading configuration") if err := deployment.LoadPlan(ctx, s); err != nil { diff --git a/dagger/solver.go b/dagger/solver.go index c20fef27..0e313081 100644 --- a/dagger/solver.go +++ b/dagger/solver.go @@ -14,6 +14,7 @@ import ( bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" + "github.com/moby/buildkit/solver/pb" bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" @@ -23,22 +24,50 @@ type Solver struct { events chan *bk.SolveStatus control *bk.Client gw bkgw.Client + noCache bool } -func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus) Solver { +func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, noCache bool) Solver { return Solver{ events: events, control: control, gw: gw, + noCache: noCache, } } +func invalidateCache(def *llb.Definition) error { + for _, dt := range def.Def { + var op pb.Op + if err := (&op).Unmarshal(dt); err != nil { + return err + } + dgst := digest.FromBytes(dt) + opMetadata, ok := def.Metadata[dgst] + if !ok { + opMetadata = pb.OpMetadata{} + } + c := llb.Constraints{Metadata: opMetadata} + llb.IgnoreCache(&c) + def.Metadata[dgst] = c.Metadata + } + + return nil +} + func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) { // FIXME: do not hardcode the platform def, err := st.Marshal(ctx, llb.LinuxAmd64) if err != nil { return nil, err } + + if s.noCache { + if err := invalidateCache(def); err != nil { + return nil, err + } + } + return def.ToPB(), nil } From bbeff0eddbaea2ce1551b588094685b289c7a921 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 14 Apr 2021 16:21:45 -0700 Subject: [PATCH 2/6] cmd: plugged no-cache option to all relevant commands Signed-off-by: Sam Alba --- cmd/dagger/cmd/common/common.go | 4 ++-- cmd/dagger/cmd/compute.go | 3 ++- cmd/dagger/cmd/new.go | 2 +- cmd/dagger/cmd/query.go | 2 +- cmd/dagger/cmd/up.go | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 83e17d70..5028a2b1 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -60,10 +60,10 @@ func GetCurrentDeploymentState(ctx context.Context, store *dagger.Store) *dagger } // Re-compute a deployment (equivalent to `dagger up`). -func DeploymentUp(ctx context.Context, state *dagger.DeploymentState) *dagger.Deployment { +func DeploymentUp(ctx context.Context, state *dagger.DeploymentState, noCache bool) *dagger.Deployment { lg := log.Ctx(ctx) - c, err := dagger.NewClient(ctx, "") + c, err := dagger.NewClient(ctx, "", noCache) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } diff --git a/cmd/dagger/cmd/compute.go b/cmd/dagger/cmd/compute.go index 9f2d32e4..4596315f 100644 --- a/cmd/dagger/cmd/compute.go +++ b/cmd/dagger/cmd/compute.go @@ -149,7 +149,7 @@ var computeCmd = &cobra.Command{ } } - deployment := common.DeploymentUp(ctx, st) + deployment := common.DeploymentUp(ctx, st, viper.GetBool("no-cache")) v := compiler.NewValue() if err := v.FillPath(cue.MakePath(), deployment.Plan()); err != nil { @@ -173,6 +173,7 @@ func init() { computeCmd.Flags().StringSlice("input-git", []string{}, "TARGET=REMOTE#REF") computeCmd.Flags().String("input-json", "", "JSON") computeCmd.Flags().String("input-yaml", "", "YAML") + computeCmd.Flags().Bool("no-cache", false, "disable cache") if err := viper.BindPFlags(computeCmd.Flags()); err != nil { panic(err) diff --git a/cmd/dagger/cmd/new.go b/cmd/dagger/cmd/new.go index 90d68468..9c3e3966 100644 --- a/cmd/dagger/cmd/new.go +++ b/cmd/dagger/cmd/new.go @@ -63,7 +63,7 @@ var newCmd = &cobra.Command{ Msg("deployment created") if viper.GetBool("up") { - common.DeploymentUp(ctx, st) + common.DeploymentUp(ctx, st, false) } }, } diff --git a/cmd/dagger/cmd/query.go b/cmd/dagger/cmd/query.go index 5680371f..3a1ed73a 100644 --- a/cmd/dagger/cmd/query.go +++ b/cmd/dagger/cmd/query.go @@ -47,7 +47,7 @@ var queryCmd = &cobra.Command{ cuePath = cue.ParsePath(args[0]) } - c, err := dagger.NewClient(ctx, "") + c, err := dagger.NewClient(ctx, "", false) if err != nil { lg.Fatal().Err(err).Msg("unable to create client") } diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 7d6175e8..7e5a23e9 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -31,7 +31,7 @@ var upCmd = &cobra.Command{ state := common.GetCurrentDeploymentState(ctx, store) // TODO: Implement options: --no-cache - result := common.DeploymentUp(ctx, state) + result := common.DeploymentUp(ctx, state, viper.GetBool("no-cache")) state.Computed = result.Computed().JSON().String() if err := store.UpdateDeployment(ctx, state, nil); err != nil { lg.Fatal().Err(err).Msg("failed to update deployment") @@ -40,7 +40,7 @@ var upCmd = &cobra.Command{ } func init() { - newCmd.Flags().Bool("--no-cache", false, "Disable all run cache") + upCmd.Flags().Bool("no-cache", false, "Disable all run cache") if err := viper.BindPFlags(upCmd.Flags()); err != nil { panic(err) From 7ed616c7725a350b23d476809d62fcde88ad9dcb Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 14 Apr 2021 16:25:10 -0700 Subject: [PATCH 3/6] cmd: fixed typo and removed solved TODO msg Signed-off-by: Sam Alba --- cmd/dagger/cmd/down.go | 2 +- cmd/dagger/cmd/up.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/dagger/cmd/down.go b/cmd/dagger/cmd/down.go index 1e3937ac..5766c8b5 100644 --- a/cmd/dagger/cmd/down.go +++ b/cmd/dagger/cmd/down.go @@ -22,7 +22,7 @@ var downCmd = &cobra.Command{ } func init() { - downCmd.Flags().Bool("--no-cache", false, "Disable all run cache") + downCmd.Flags().Bool("no-cache", false, "Disable all run cache") if err := viper.BindPFlags(downCmd.Flags()); err != nil { panic(err) diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 7e5a23e9..b0ecf81f 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -29,8 +29,6 @@ var upCmd = &cobra.Command{ } state := common.GetCurrentDeploymentState(ctx, store) - - // TODO: Implement options: --no-cache result := common.DeploymentUp(ctx, state, viper.GetBool("no-cache")) state.Computed = result.Computed().JSON().String() if err := store.UpdateDeployment(ctx, state, nil); err != nil { From 9ffe5c053c1fad61b5c7abb8621ba6d46acf2130 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 14 Apr 2021 16:27:15 -0700 Subject: [PATCH 4/6] solver: fixed lint error (double import) Signed-off-by: Sam Alba --- dagger/solver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dagger/solver.go b/dagger/solver.go index 0e313081..3a78c0c1 100644 --- a/dagger/solver.go +++ b/dagger/solver.go @@ -14,7 +14,6 @@ import ( bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" - "github.com/moby/buildkit/solver/pb" bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" @@ -38,14 +37,14 @@ func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, func invalidateCache(def *llb.Definition) error { for _, dt := range def.Def { - var op pb.Op + var op bkpb.Op if err := (&op).Unmarshal(dt); err != nil { return err } dgst := digest.FromBytes(dt) opMetadata, ok := def.Metadata[dgst] if !ok { - opMetadata = pb.OpMetadata{} + opMetadata = bkpb.OpMetadata{} } c := llb.Constraints{Metadata: opMetadata} llb.IgnoreCache(&c) From 075fbbf3d585ab3c56febbc73a8ccef7a5b2d9e6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 11:26:01 -0700 Subject: [PATCH 5/6] pipeline: forward the no-cache option to the dockerbuild frontend Signed-off-by: Sam Alba --- dagger/pipeline.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dagger/pipeline.go b/dagger/pipeline.go index 7cddcfd0..271dbb5d 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -722,6 +722,10 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S return st, err } + if p.s.noCache { + opts["no-cache"] = "" + } + req := bkgw.SolveRequest{ Frontend: "dockerfile.v0", FrontendOpt: opts, From ee555dea0c068f6b881518d62e7ffe2feb2d8ca6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 15 Apr 2021 11:26:21 -0700 Subject: [PATCH 6/6] examples/jamstack: fixed logic to test the push-container with no-cache Signed-off-by: Sam Alba --- examples/jamstack/ecr_image.cue | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/examples/jamstack/ecr_image.cue b/examples/jamstack/ecr_image.cue index 81d44fe6..20963737 100644 --- a/examples/jamstack/ecr_image.cue +++ b/examples/jamstack/ecr_image.cue @@ -19,15 +19,6 @@ import ( pushTarget: "\(repository):\(tag)" - // Build the image - buildImage: op.#DockerBuild & { - context: source - if dockerfilePath != _|_ { - "dockerfilePath": dockerfilePath - } - buildArg: buildArgs - } - // Use these credentials to push ecrCreds: ecr.#Credentials & { config: awsConfig @@ -35,6 +26,7 @@ import ( } push: #up: [ + // Build the docker image op.#DockerBuild & { context: source if dockerfilePath != _|_ { @@ -42,6 +34,7 @@ import ( } buildArg: buildArgs }, + // Push the image to the registry op.#PushContainer & { ref: pushTarget },