From 19c0f999f45bde711e623213b9dc07c6d08b3c33 Mon Sep 17 00:00:00 2001 From: Tom Chauveau Date: Wed, 23 Mar 2022 23:02:17 +0100 Subject: [PATCH] Fix export cache issue Resolve #1551 and #1020. We are never returning the result of solved operations so Buildkit could not cache the layer. This commit implements a simple system to forward operations' result to the main build to cache it. Signed-off-by: Vasek - Tom C --- .github/workflows/dagger-ci.yml | 11 ++--- .github/workflows/test-integration.yml | 9 ++-- .github/workflows/test-universe.yml | 3 -- Makefile | 2 +- client/client.go | 44 +++++++++++------ cmd/dagger/cmd/do.go | 2 +- docs/learn/tests/helpers.bash | 6 +++ pkg/universe.dagger.io/bats_helpers.bash | 7 +++ pkg/universe.dagger.io/package.json | 2 +- plan/plan.go | 2 +- plan/runner.go | 4 +- plan/task/clientcommand.go | 2 +- plan/task/clientenv.go | 2 +- plan/task/clientfilesystemread.go | 8 ++-- plan/task/clientfilesystemwrite.go | 6 +-- plan/task/clientnetwork.go | 2 +- plan/task/clientplatform.go | 2 +- plan/task/copy.go | 2 +- plan/task/decodesecret.go | 2 +- plan/task/diff.go | 2 +- plan/task/dockerfile.go | 2 +- plan/task/exec.go | 8 ++-- plan/task/export.go | 4 +- plan/task/gitpull.go | 2 +- plan/task/httpfetch.go | 2 +- plan/task/merge.go | 2 +- plan/task/mkdir.go | 2 +- plan/task/newsecret.go | 2 +- plan/task/nop.go | 2 +- plan/task/pull.go | 4 +- plan/task/push.go | 2 +- plan/task/readfile.go | 2 +- plan/task/source.go | 4 +- plan/task/task.go | 2 +- plan/task/transformsecret.go | 2 +- plan/task/trimsecret.go | 2 +- plan/task/writefile.go | 7 +-- solver/solver.go | 61 ++++++++++++++++-------- tests/helpers.bash | 7 +++ tests/package.json | 2 +- tests/plan.bats | 4 +- tests/project.bats | 2 +- 42 files changed, 144 insertions(+), 103 deletions(-) diff --git a/.github/workflows/dagger-ci.yml b/.github/workflows/dagger-ci.yml index 03d3f90b..1492b19e 100644 --- a/.github/workflows/dagger-ci.yml +++ b/.github/workflows/dagger-ci.yml @@ -2,7 +2,7 @@ name: "Dagger CI" on: push: - branches: [main] + branches: [ main ] paths: - '**.sh' - '**.bash' @@ -14,7 +14,7 @@ on: - 'go.sum' - '.github/workflows/dagger-ci.yml' pull_request: - branches: [main] + branches: [ main ] paths: - '**.sh' - '**.bash' @@ -33,11 +33,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - - name: Dagger CI + + - name: Dagger CI uses: dagger/dagger-for-github@v2 with: workdir: ci diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index e6c4d84c..78332524 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -2,7 +2,7 @@ name: "Test Integration" on: push: - branches: [main] + branches: [ main ] paths: - "**.sh" - "**.bash" @@ -16,7 +16,7 @@ on: - "!docs/**" pull_request: - branches: [main] + branches: [ main ] paths: - "**.sh" - "**.bash" @@ -67,9 +67,6 @@ jobs: - name: Test env: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - # TODO: https://github.com/dagger/dagger/pull/1341 - # DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-integration" - # DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-integration" run: | env - make core-integration + make core-integration \ No newline at end of file diff --git a/.github/workflows/test-universe.yml b/.github/workflows/test-universe.yml index 212a273d..dcfb1b2c 100644 --- a/.github/workflows/test-universe.yml +++ b/.github/workflows/test-universe.yml @@ -57,8 +57,5 @@ jobs: uses: crazy-max/ghaction-github-runtime@v1 - name: Test - env: - DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-universe" - DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-universe" run: | make universe-test diff --git a/Makefile b/Makefile index 4cf048ec..31d3d09b 100644 --- a/Makefile +++ b/Makefile @@ -93,4 +93,4 @@ web: # Run the website locally .PHONY: todo todo: # Find all TODO items - grep -r -A 1 "TODO:" $(CURDIR) + grep -r -A 1 "TODO:" $(CURDIR) \ No newline at end of file diff --git a/client/client.go b/client/client.go index bd8cc25b..071ea373 100644 --- a/client/client.go +++ b/client/client.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/containerd/containerd/platforms" + "github.com/google/uuid" "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" @@ -18,7 +18,6 @@ import ( // buildkit bk "github.com/moby/buildkit/client" _ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver - "github.com/moby/buildkit/client/llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" @@ -72,7 +71,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) { }, nil } -type DoFunc func(context.Context, solver.Solver) error +type DoFunc func(context.Context, *solver.Solver) error // FIXME: return completed *Route, instead of *compiler.Value func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error { @@ -96,6 +95,19 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) e return eg.Wait() } +func convertCacheOptionEntries(ims []bk.CacheOptionsEntry) []bkgw.CacheOptionsEntry { + convertIms := []bkgw.CacheOptionsEntry{} + + for _, im := range ims { + convertIm := bkgw.CacheOptionsEntry{ + Type: im.Type, + Attrs: im.Attrs, + } + convertIms = append(convertIms, convertIm) + } + return convertIms +} + func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error { wg := sync.WaitGroup{} @@ -156,29 +168,31 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFu resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { s := solver.New(solver.Opts{ - Control: c.c, - Gateway: gw, - Events: eventsCh, - Auth: auth, - NoCache: c.cfg.NoCache, + Control: c.c, + Gateway: gw, + Events: eventsCh, + Auth: auth, + NoCache: c.cfg.NoCache, + CacheImports: convertCacheOptionEntries(opts.CacheImports), }) // Close events channel defer s.Stop() // Compute output overlay + res := bkgw.NewResult() if fn != nil { - if err := fn(ctx, s); err != nil { + err := fn(ctx, s) + if err != nil { return nil, compiler.Err(err) } - } - ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec()) - if err != nil { - return nil, err + refs := s.References() + // Add functions layers + for _, ref := range refs { + res.AddRef(uuid.New().String(), ref) + } } - res := bkgw.NewResult() - res.SetRef(ref) return res, nil }, buildCh) if err != nil { diff --git a/cmd/dagger/cmd/do.go b/cmd/dagger/cmd/do.go index ff6f6a27..c11c2313 100644 --- a/cmd/dagger/cmd/do.go +++ b/cmd/dagger/cmd/do.go @@ -67,7 +67,7 @@ var doCmd = &cobra.Command{ Value: target.String(), }) - err = cl.Do(ctx, p.Context(), func(ctx context.Context, s solver.Solver) error { + err = cl.Do(ctx, p.Context(), func(ctx context.Context, s *solver.Solver) error { return p.Do(ctx, target, s) }) diff --git a/docs/learn/tests/helpers.bash b/docs/learn/tests/helpers.bash index 96408a91..4eaf18ea 100644 --- a/docs/learn/tests/helpers.bash +++ b/docs/learn/tests/helpers.bash @@ -28,6 +28,12 @@ common_setup() { export DAGGER_SANDBOX dagger init --project "$DAGGER_SANDBOX" + if [ -n "$GITHUB_ACTIONS" ]; + then + export DAGGER_CACHE_TO="type=gha,mode=max,scope=docs-tests-$BATS_TEST_NAME" + export DAGGER_CACHE_FROM="type=gha,scope=docs-tests-$BATS_TEST_NAME" + fi + # allows the use of `sops` SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt export SOPS_AGE_KEY_FILE diff --git a/pkg/universe.dagger.io/bats_helpers.bash b/pkg/universe.dagger.io/bats_helpers.bash index b8609ebe..d38d8876 100644 --- a/pkg/universe.dagger.io/bats_helpers.bash +++ b/pkg/universe.dagger.io/bats_helpers.bash @@ -15,6 +15,13 @@ common_setup() { DAGGER_LOG_FORMAT="plain" export DAGGER_LOG_FORMAT + export DAGGER_LOG_LEVEL="debug" + if [ -n "$GITHUB_ACTIONS" ]; + then + export DAGGER_CACHE_TO="type=gha,mode=max,scope=universe-tests-$BATS_TEST_NAME" + export DAGGER_CACHE_FROM="type=gha,scope=universe-tests-$BATS_TEST_NAME" + fi + # cd into the directory containing the bats file cd "$BATS_TEST_DIRNAME" || exit 1 } diff --git a/pkg/universe.dagger.io/package.json b/pkg/universe.dagger.io/package.json index bc23fdf5..a6d7bbba 100644 --- a/pkg/universe.dagger.io/package.json +++ b/pkg/universe.dagger.io/package.json @@ -1,7 +1,7 @@ { "license": "Apache-2.0", "scripts": { - "test": "bats --report-formatter junit --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')" + "test": "bats --report-formatter junit --print-output-on-failure --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')" }, "devDependencies": { "bats": "^1.5.0", diff --git a/plan/plan.go b/plan/plan.go index 04adf17a..dc4cc0df 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -168,7 +168,7 @@ func (p *Plan) prepare(ctx context.Context) error { } // Do executes an action in the plan -func (p *Plan) Do(ctx context.Context, path cue.Path, s solver.Solver) error { +func (p *Plan) Do(ctx context.Context, path cue.Path, s *solver.Solver) error { ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up") defer span.End() diff --git a/plan/runner.go b/plan/runner.go index 219f5fbe..a59e80ac 100644 --- a/plan/runner.go +++ b/plan/runner.go @@ -22,13 +22,13 @@ import ( type Runner struct { pctx *plancontext.Context target cue.Path - s solver.Solver + s *solver.Solver tasks sync.Map mirror *compiler.Value l sync.Mutex } -func NewRunner(pctx *plancontext.Context, target cue.Path, s solver.Solver) *Runner { +func NewRunner(pctx *plancontext.Context, target cue.Path, s *solver.Solver) *Runner { return &Runner{ pctx: pctx, target: target, diff --git a/plan/task/clientcommand.go b/plan/task/clientcommand.go index 50858c67..eab1f0bf 100644 --- a/plan/task/clientcommand.go +++ b/plan/task/clientcommand.go @@ -23,7 +23,7 @@ func init() { type clientCommandTask struct { } -func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { var opts struct { Name string Args []string diff --git a/plan/task/clientenv.go b/plan/task/clientenv.go index 5b51c483..52fe75db 100644 --- a/plan/task/clientenv.go +++ b/plan/task/clientenv.go @@ -19,7 +19,7 @@ func init() { type clientEnvTask struct { } -func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { log.Ctx(ctx).Debug().Msg("loading environment variables") fields, err := v.Fields() diff --git a/plan/task/clientfilesystemread.go b/plan/task/clientfilesystemread.go index 792a217b..0ec37f1a 100644 --- a/plan/task/clientfilesystemread.go +++ b/plan/task/clientfilesystemread.go @@ -21,7 +21,7 @@ func init() { type clientFilesystemReadTask struct { } -func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { +func (t clientFilesystemReadTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error { path, err := t.parsePath(v) if err != nil { return err @@ -38,7 +38,7 @@ func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext. return nil } -func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { path, err := t.parsePath(v) if err != nil { return nil, err @@ -70,7 +70,7 @@ func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err return } -func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) { +func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (interface{}, error) { lg := log.Ctx(ctx) contents := v.Lookup("contents") @@ -97,7 +97,7 @@ func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *planco return nil, fmt.Errorf("unsupported type %q", k) } -func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) { +func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) { var dir struct { Include []string Exclude []string diff --git a/plan/task/clientfilesystemwrite.go b/plan/task/clientfilesystemwrite.go index 1ccbb325..dc0994b1 100644 --- a/plan/task/clientfilesystemwrite.go +++ b/plan/task/clientfilesystemwrite.go @@ -21,7 +21,7 @@ func init() { type clientFilesystemWriteTask struct { } -func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { path, err := v.Lookup("path").String() if err != nil { return nil, err @@ -39,7 +39,7 @@ func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Co return compiler.NewValue(), nil } -func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { +func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error { lg := log.Ctx(ctx) contents := v.Lookup("contents") @@ -79,7 +79,7 @@ func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plan return fmt.Errorf("unsupported type %q", k) } -func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { +func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error { contents, err := pctx.FS.FromValue(v) if err != nil { return err diff --git a/plan/task/clientnetwork.go b/plan/task/clientnetwork.go index 7b1330b8..270cb7b7 100644 --- a/plan/task/clientnetwork.go +++ b/plan/task/clientnetwork.go @@ -20,7 +20,7 @@ func init() { type clientNetwork struct { } -func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) addr, err := v.Lookup("address").String() diff --git a/plan/task/clientplatform.go b/plan/task/clientplatform.go index 4fc2ad06..a2bc1169 100644 --- a/plan/task/clientplatform.go +++ b/plan/task/clientplatform.go @@ -16,7 +16,7 @@ func init() { type clientPlatformTask struct { } -func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t clientPlatformTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, _ *compiler.Value) (*compiler.Value, error) { return compiler.NewValue().FillFields(map[string]interface{}{ "os": runtime.GOOS, "arch": runtime.GOARCH, diff --git a/plan/task/copy.go b/plan/task/copy.go index dc6b8995..172cb064 100644 --- a/plan/task/copy.go +++ b/plan/task/copy.go @@ -16,7 +16,7 @@ func init() { type copyTask struct { } -func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { var err error input, err := pctx.FS.FromValue(v.Lookup("input")) diff --git a/plan/task/decodesecret.go b/plan/task/decodesecret.go index c7e9906d..2c0a4aa9 100644 --- a/plan/task/decodesecret.go +++ b/plan/task/decodesecret.go @@ -20,7 +20,7 @@ func init() { type decodeSecretTask struct { } -func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) lg.Debug().Msg("decoding secret") diff --git a/plan/task/diff.go b/plan/task/diff.go index 5adbf709..fcb1b256 100644 --- a/plan/task/diff.go +++ b/plan/task/diff.go @@ -16,7 +16,7 @@ func init() { type diffTask struct { } -func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *diffTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lowerFS, err := pctx.FS.FromValue(v.Lookup("lower")) if err != nil { return nil, err diff --git a/plan/task/dockerfile.go b/plan/task/dockerfile.go index 1254114f..4a8f1712 100644 --- a/plan/task/dockerfile.go +++ b/plan/task/dockerfile.go @@ -28,7 +28,7 @@ func init() { type dockerfileTask struct { } -func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) auths, err := v.Lookup("auth").Fields() if err != nil { diff --git a/plan/task/exec.go b/plan/task/exec.go index c5e3f16e..3f9b3d9e 100644 --- a/plan/task/exec.go +++ b/plan/task/exec.go @@ -20,7 +20,7 @@ func init() { type execTask struct { } -func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *execTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { // Get input state input, err := pctx.FS.FromValue(v.Lookup("input")) if err != nil { @@ -52,7 +52,7 @@ func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.S }) } -func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) { +func (t *execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) { opts := []llb.RunOption{} var cmd struct { Args []string @@ -141,7 +141,7 @@ func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]ll return opts, nil } -func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) { +func (t *execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) { opts := []llb.RunOption{} fields, err := mounts.Fields() if err != nil { @@ -165,7 +165,7 @@ func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([ return opts, err } -func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { +func (t *execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { typ, err := mnt.Lookup("type").String() if err != nil { return nil, err diff --git a/plan/task/export.go b/plan/task/export.go index 92c7d9df..53f557e4 100644 --- a/plan/task/export.go +++ b/plan/task/export.go @@ -25,7 +25,7 @@ func init() { type exportTask struct { } -func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { +func (t exportTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error { dir, err := os.MkdirTemp("", "dagger-export-*") if err != nil { return err @@ -37,7 +37,7 @@ func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *co return nil } -func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) dir := pctx.TempDirs.Get(v.Path().String()) diff --git a/plan/task/gitpull.go b/plan/task/gitpull.go index cb8532bd..a3dc5738 100644 --- a/plan/task/gitpull.go +++ b/plan/task/gitpull.go @@ -19,7 +19,7 @@ func init() { type gitPullTask struct { } -func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { var gitPull struct { Remote string Ref string diff --git a/plan/task/httpfetch.go b/plan/task/httpfetch.go index d70caffe..8a8e68fd 100644 --- a/plan/task/httpfetch.go +++ b/plan/task/httpfetch.go @@ -21,7 +21,7 @@ func init() { type httpFetchTask struct { } -func (c httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { var httpFetch struct { Source string Checksum string diff --git a/plan/task/merge.go b/plan/task/merge.go index 1c5c18b1..60d62e3c 100644 --- a/plan/task/merge.go +++ b/plan/task/merge.go @@ -16,7 +16,7 @@ func init() { type mergeTask struct { } -func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { inputs, err := v.Lookup("inputs").List() if err != nil { return nil, err diff --git a/plan/task/mkdir.go b/plan/task/mkdir.go index 04e2f40c..c887ead2 100644 --- a/plan/task/mkdir.go +++ b/plan/task/mkdir.go @@ -18,7 +18,7 @@ func init() { type mkdirTask struct { } -func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { path, err := v.Lookup("path").String() if err != nil { return nil, err diff --git a/plan/task/newsecret.go b/plan/task/newsecret.go index 0fbaf8ec..cd749bad 100644 --- a/plan/task/newsecret.go +++ b/plan/task/newsecret.go @@ -18,7 +18,7 @@ func init() { type newSecretTask struct { } -func (t *newSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *newSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { path, err := v.Lookup("path").String() if err != nil { return nil, err diff --git a/plan/task/nop.go b/plan/task/nop.go index 8a7dfc81..81f4c5f9 100644 --- a/plan/task/nop.go +++ b/plan/task/nop.go @@ -15,6 +15,6 @@ func init() { type nopTask struct { } -func (t *nopTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *nopTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { return v, nil } diff --git a/plan/task/pull.go b/plan/task/pull.go index d374ff5c..88e179ba 100644 --- a/plan/task/pull.go +++ b/plan/task/pull.go @@ -19,7 +19,7 @@ func init() { type pullTask struct { } -func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) rawRef, err := v.Lookup("source").String() @@ -68,8 +68,8 @@ func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver. if err != nil { return nil, err } - fs := pctx.FS.New(result) + fs := pctx.FS.New(result) return compiler.NewValue().FillFields(map[string]interface{}{ "output": fs.MarshalCUE(), "digest": digest, diff --git a/plan/task/push.go b/plan/task/push.go index 1adf5546..2ce8b64c 100644 --- a/plan/task/push.go +++ b/plan/task/push.go @@ -19,7 +19,7 @@ func init() { type pushTask struct { } -func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) rawDest, err := v.Lookup("dest").String() diff --git a/plan/task/readfile.go b/plan/task/readfile.go index cb6a1f25..6d31d06d 100644 --- a/plan/task/readfile.go +++ b/plan/task/readfile.go @@ -18,7 +18,7 @@ func init() { type readFileTask struct { } -func (t *readFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *readFileTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { path, err := v.Lookup("path").String() if err != nil { return nil, err diff --git a/plan/task/source.go b/plan/task/source.go index 50062f84..19556b39 100644 --- a/plan/task/source.go +++ b/plan/task/source.go @@ -21,7 +21,7 @@ func init() { type sourceTask struct { } -func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { +func (c *sourceTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error { origPath, err := v.Lookup("path").String() if err != nil { return err @@ -50,7 +50,7 @@ func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *c return nil } -func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) path, err := v.Lookup("path").AbsPath() diff --git a/plan/task/task.go b/plan/task/task.go index 937a4a9b..1682144f 100644 --- a/plan/task/task.go +++ b/plan/task/task.go @@ -40,7 +40,7 @@ const ( type NewFunc func() Task type Task interface { - Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) + Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) } type PreRunner interface { diff --git a/plan/task/transformsecret.go b/plan/task/transformsecret.go index 3bc482ea..4b1993b1 100644 --- a/plan/task/transformsecret.go +++ b/plan/task/transformsecret.go @@ -20,7 +20,7 @@ func init() { type transformSecretTask struct { } -func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { lg := log.Ctx(ctx) lg.Debug().Msg("transforming secret") diff --git a/plan/task/trimsecret.go b/plan/task/trimsecret.go index f4441d9d..6882a98a 100644 --- a/plan/task/trimsecret.go +++ b/plan/task/trimsecret.go @@ -16,7 +16,7 @@ func init() { type trimSecretTask struct { } -func (t *trimSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *trimSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) { input, err := pctx.Secrets.FromValue(v.Lookup("input")) if err != nil { return nil, err diff --git a/plan/task/writefile.go b/plan/task/writefile.go index 90af30c7..6bfa7843 100644 --- a/plan/task/writefile.go +++ b/plan/task/writefile.go @@ -19,7 +19,7 @@ func init() { type writeFileTask struct { } -func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { +func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) { var contents []byte var err error @@ -49,19 +49,16 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so } permissions, err := v.Lookup("permissions").Int64() - if err != nil { return nil, err } input, err := pctx.FS.FromValue(v.Lookup("input")) - if err != nil { return nil, err } inputState, err := input.State() - if err != nil { return nil, err } @@ -72,7 +69,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so ) result, err := s.Solve(ctx, outputState, pctx.Platform.Get()) - if err != nil { return nil, err } @@ -80,7 +76,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so outputFS := pctx.FS.New(result) output := compiler.NewValue() - if err := output.FillPath(cue.ParsePath("output"), outputFS.MarshalCUE()); err != nil { return nil, err } diff --git a/solver/solver.go b/solver/solver.go index 0bd2d384..56ee900c 100644 --- a/solver/solver.go +++ b/solver/solver.go @@ -25,19 +25,22 @@ type Solver struct { opts Opts eventsWg *sync.WaitGroup closeCh chan *bk.SolveStatus + refs []bkgw.Reference + l sync.RWMutex } type Opts struct { - Control *bk.Client - Gateway bkgw.Client - Events chan *bk.SolveStatus - Context *plancontext.Context - Auth *RegistryAuthProvider - NoCache bool + Control *bk.Client + Gateway bkgw.Client + Events chan *bk.SolveStatus + Context *plancontext.Context + Auth *RegistryAuthProvider + NoCache bool + CacheImports []bkgw.CacheOptionsEntry } -func New(opts Opts) Solver { - return Solver{ +func New(opts Opts) *Solver { + return &Solver{ eventsWg: &sync.WaitGroup{}, closeCh: make(chan *bk.SolveStatus), opts: opts, @@ -63,25 +66,25 @@ func invalidateCache(def *llb.Definition) error { return nil } -func (s Solver) GetOptions() Opts { +func (s *Solver) GetOptions() Opts { return s.opts } -func (s Solver) NoCache() bool { +func (s *Solver) NoCache() bool { return s.opts.NoCache } -func (s Solver) Stop() { +func (s *Solver) Stop() { close(s.closeCh) s.eventsWg.Wait() close(s.opts.Events) } -func (s Solver) AddCredentials(target, username, secret string) { +func (s *Solver) AddCredentials(target, username, secret string) { s.opts.Auth.AddCredentials(target, username, secret) } -func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) { +func (s *Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) { // FIXME: do not hardcode the platform def, err := st.Marshal(ctx, co...) if err != nil { @@ -97,11 +100,11 @@ func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.Constraints return def.ToPB(), nil } -func (s Solver) SessionID() string { +func (s *Solver) SessionID() string { return s.opts.Gateway.BuildOpts().SessionID } -func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) { +func (s *Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) { var image dockerfile2llb.Image // Load image metadata and convert to to LLB. @@ -119,7 +122,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res } // Solve will block until the state is solved and returns a Reference. -func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) { +func (s *Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) { // makes Solve() to block until LLB graph is solved. otherwise it will // return result (that you can for example use for next build) that // will be evaluated on export or if you access files on it. @@ -131,9 +134,15 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw. return res, nil } +func (s *Solver) References() []bkgw.Reference { + s.l.RLock() + defer s.l.RUnlock() + return s.refs +} + // Solve will block until the state is solved and returns a Reference. // It takes a platform as argument which correspond to the targeted platform. -func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) { +func (s *Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) { def, err := s.Marshal(ctx, st, llb.Platform(platform)) if err != nil { return nil, err @@ -152,19 +161,29 @@ func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform // call solve res, err := s.SolveRequest(ctx, bkgw.SolveRequest{ - Definition: def, + Definition: def, + CacheImports: s.opts.CacheImports, }) if err != nil { return nil, err } - return res.SingleRef() + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + s.l.Lock() + defer s.l.Unlock() + s.refs = append(s.refs, ref) + + return ref, nil } // Forward events from solver to the main events channel // It creates a task in the solver waiting group to be // sure that everything will be forward to the main channel -func (s Solver) forwardEvents(ch chan *bk.SolveStatus) { +func (s *Solver) forwardEvents(ch chan *bk.SolveStatus) { s.eventsWg.Add(1) defer s.eventsWg.Done() @@ -177,7 +196,7 @@ func (s Solver) forwardEvents(ch chan *bk.SolveStatus) { // FIXME: this is currently implemented as a hack, starting a new Build session // within buildkit from the Control API. Ideally the Gateway API should allow to // Export directly. -func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) { +func (s *Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) { // Check close event channel and return if we're already done with the main pipeline select { case <-s.closeCh: diff --git a/tests/helpers.bash b/tests/helpers.bash index b5bcba91..3a90421d 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -13,6 +13,13 @@ common_setup() { DAGGER_TELEMETRY_DISABLE="1" export DAGGER_TELEMETRY_DISABLE + export DAGGER_LOG_LEVEL="debug" + if [ -n "$GITHUB_ACTIONS" ]; + then + export DAGGER_CACHE_TO="type=gha,mode=max,scope=integration-tests-$BATS_TEST_NAME" + export DAGGER_CACHE_FROM="type=gha,scope=integration-tests-$BATS_TEST_NAME" + fi + SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt export SOPS_AGE_KEY_FILE } diff --git a/tests/package.json b/tests/package.json index 94e0d7b4..b959fad7 100644 --- a/tests/package.json +++ b/tests/package.json @@ -1,7 +1,7 @@ { "license": "Apache-2.0", "scripts": { - "test": "bats --jobs 4 --show-output-of-passing-tests --print-output-on-failure ." + "test": "bats --jobs 4 --print-output-on-failure --verbose-run ." }, "devDependencies": { "bats": "https://github.com/bats-core/bats-core#master", diff --git a/tests/plan.bats b/tests/plan.bats index 8af74b72..c783daa9 100644 --- a/tests/plan.bats +++ b/tests/plan.bats @@ -144,7 +144,7 @@ setup() { cd "$TESTDIR/plan/client/filesystem/conflict" echo -n foo > test.txt - run "$DAGGER" "do" --log-level debug -p . test + run "$DAGGER" "do" -p . test assert_line --regexp "client\.filesystem\..+\.write.+dependency=client\.filesystem\..+\.read" rm -f test.txt @@ -221,4 +221,4 @@ setup() { # Run with invalid platform run "$DAGGER" "do" -p./plan/platform/config_platform_failure_invalid_platform.cue verify assert_failure -} +} \ No newline at end of file diff --git a/tests/project.bats b/tests/project.bats index 111c073e..65574fe3 100644 --- a/tests/project.bats +++ b/tests/project.bats @@ -25,6 +25,6 @@ setup() { test -f ./cue.mod/pkg/.gitattributes run cat ./cue.mod/pkg/.gitattributes assert_output --partial "generated by dagger" - + test ! -f ./cue.mod/pkg/.gitignore }