kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Fix export cache issue

Browse Source 
Resolve #1551 and #1020.
We are never returning the result of solved operations so Buildkit could not
cache the layer.
This commit implements a simple system to forward operations' result to the
main build to cache it.

Signed-off-by: Vasek - Tom C <tom.chauveau@epitech.eu>
This commit is contained in:
Tom Chauveau 2022-03-23 23:02:17 +01:00 committed by Vasek - Tom C
parent 64cdadb85e
commit 19c0f999f4
No known key found for this signature in database
GPG Key ID: 175D82E572427960
42 changed files with 144 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/dagger-ci.yml vendored
View File

@ -33,11 +33,10 @@ jobs:
build: build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- - name: Checkout
name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
-
name: Dagger CI - name: Dagger CI
uses: dagger/dagger-for-github@v2 uses: dagger/dagger-for-github@v2
with: with:
workdir: ci workdir: ci

3
.github/workflows/test-integration.yml vendored
View File

@ -67,9 +67,6 @@ jobs:
- name: Test - name: Test
env: env:
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
# TODO: https://github.com/dagger/dagger/pull/1341
# DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-integration"
# DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-integration"
run: | run: |
env env
make core-integration make core-integration

3
.github/workflows/test-universe.yml vendored
View File

@ -57,8 +57,5 @@ jobs:
uses: crazy-max/ghaction-github-runtime@v1 uses: crazy-max/ghaction-github-runtime@v1
- name: Test - name: Test
env:
DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-universe"
DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-universe"
run: | run: |
make universe-test make universe-test

34
client/client.go
View File

@ -7,7 +7,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/containerd/containerd/platforms" "github.com/google/uuid"
"go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
@ -18,7 +18,6 @@ import (
// buildkit // buildkit
bk "github.com/moby/buildkit/client" bk "github.com/moby/buildkit/client"
_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver _ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client" bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session" "github.com/moby/buildkit/session"
@ -72,7 +71,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) {
}, nil }, nil
} }
type DoFunc func(context.Context, solver.Solver) error type DoFunc func(context.Context, *solver.Solver) error
// FIXME: return completed *Route, instead of *compiler.Value // FIXME: return completed *Route, instead of *compiler.Value
func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error { func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error {
@ -96,6 +95,19 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) e
return eg.Wait() return eg.Wait()
} }
func convertCacheOptionEntries(ims []bk.CacheOptionsEntry) []bkgw.CacheOptionsEntry {
convertIms := []bkgw.CacheOptionsEntry{}
for _, im := range ims {
convertIm := bkgw.CacheOptionsEntry{
Type: im.Type,
Attrs: im.Attrs,
}
convertIms = append(convertIms, convertIm)
}
return convertIms
}
func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error { func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error {
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
@ -161,24 +173,26 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFu
Events: eventsCh, Events: eventsCh,
Auth: auth, Auth: auth,
NoCache: c.cfg.NoCache, NoCache: c.cfg.NoCache,
CacheImports: convertCacheOptionEntries(opts.CacheImports),
}) })
// Close events channel // Close events channel
defer s.Stop() defer s.Stop()
// Compute output overlay // Compute output overlay
res := bkgw.NewResult()
if fn != nil { if fn != nil {
if err := fn(ctx, s); err != nil { err := fn(ctx, s)
if err != nil {
return nil, compiler.Err(err) return nil, compiler.Err(err)
} }
}
ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec()) refs := s.References()
if err != nil { // Add functions layers
return nil, err for _, ref := range refs {
res.AddRef(uuid.New().String(), ref)
}
} }
res := bkgw.NewResult()
res.SetRef(ref)
return res, nil return res, nil
}, buildCh) }, buildCh)
if err != nil { if err != nil {

2
cmd/dagger/cmd/do.go
View File

@ -67,7 +67,7 @@ var doCmd = &cobra.Command{
Value: target.String(), Value: target.String(),
}) })
err = cl.Do(ctx, p.Context(), func(ctx context.Context, s solver.Solver) error { err = cl.Do(ctx, p.Context(), func(ctx context.Context, s *solver.Solver) error {
return p.Do(ctx, target, s) return p.Do(ctx, target, s)
}) })

6
docs/learn/tests/helpers.bash
View File

@ -28,6 +28,12 @@ common_setup() {
export DAGGER_SANDBOX export DAGGER_SANDBOX
dagger init --project "$DAGGER_SANDBOX" dagger init --project "$DAGGER_SANDBOX"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=docs-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=docs-tests-$BATS_TEST_NAME"
fi
# allows the use of `sops` # allows the use of `sops`
SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt
export SOPS_AGE_KEY_FILE export SOPS_AGE_KEY_FILE

7
pkg/universe.dagger.io/bats_helpers.bash
View File

@ -15,6 +15,13 @@ common_setup() {
DAGGER_LOG_FORMAT="plain" DAGGER_LOG_FORMAT="plain"
export DAGGER_LOG_FORMAT export DAGGER_LOG_FORMAT
export DAGGER_LOG_LEVEL="debug"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=universe-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=universe-tests-$BATS_TEST_NAME"
fi
# cd into the directory containing the bats file # cd into the directory containing the bats file
cd "$BATS_TEST_DIRNAME" || exit 1 cd "$BATS_TEST_DIRNAME" || exit 1
} }

2
pkg/universe.dagger.io/package.json
View File

@ -1,7 +1,7 @@
{ {
"license": "Apache-2.0", "license": "Apache-2.0",
"scripts": { "scripts": {
"test": "bats --report-formatter junit --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')" "test": "bats --report-formatter junit --print-output-on-failure --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')"
}, },
"devDependencies": { "devDependencies": {
"bats": "^1.5.0", "bats": "^1.5.0",

2
plan/plan.go
View File

@ -168,7 +168,7 @@ func (p *Plan) prepare(ctx context.Context) error {
} }
// Do executes an action in the plan // Do executes an action in the plan
func (p *Plan) Do(ctx context.Context, path cue.Path, s solver.Solver) error { func (p *Plan) Do(ctx context.Context, path cue.Path, s *solver.Solver) error {
ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up") ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up")
defer span.End() defer span.End()

4
plan/runner.go
View File

@ -22,13 +22,13 @@ import (
type Runner struct { type Runner struct {
pctx *plancontext.Context pctx *plancontext.Context
target cue.Path target cue.Path
s solver.Solver s *solver.Solver
tasks sync.Map tasks sync.Map
mirror *compiler.Value mirror *compiler.Value
l sync.Mutex l sync.Mutex
} }
func NewRunner(pctx *plancontext.Context, target cue.Path, s solver.Solver) *Runner { func NewRunner(pctx *plancontext.Context, target cue.Path, s *solver.Solver) *Runner {
return &Runner{ return &Runner{
pctx: pctx, pctx: pctx,
target: target, target: target,

2
plan/task/clientcommand.go
View File

@ -23,7 +23,7 @@ func init() {
type clientCommandTask struct { type clientCommandTask struct {
} }
func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var opts struct { var opts struct {
Name string Name string
Args []string Args []string

2
plan/task/clientenv.go
View File

@ -19,7 +19,7 @@ func init() {
type clientEnvTask struct { type clientEnvTask struct {
} }
func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
log.Ctx(ctx).Debug().Msg("loading environment variables") log.Ctx(ctx).Debug().Msg("loading environment variables")
fields, err := v.Fields() fields, err := v.Fields()

8
plan/task/clientfilesystemread.go
View File

@ -21,7 +21,7 @@ func init() {
type clientFilesystemReadTask struct { type clientFilesystemReadTask struct {
} }
func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { func (t clientFilesystemReadTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
path, err := t.parsePath(v) path, err := t.parsePath(v)
if err != nil { if err != nil {
return err return err
@ -38,7 +38,7 @@ func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.
return nil return nil
} }
func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := t.parsePath(v) path, err := t.parsePath(v)
if err != nil { if err != nil {
return nil, err return nil, err
@ -70,7 +70,7 @@ func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err
return return
} }
func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) { func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (interface{}, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
contents := v.Lookup("contents") contents := v.Lookup("contents")
@ -97,7 +97,7 @@ func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *planco
return nil, fmt.Errorf("unsupported type %q", k) return nil, fmt.Errorf("unsupported type %q", k)
} }
func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) { func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
var dir struct { var dir struct {
Include []string Include []string
Exclude []string Exclude []string

6
plan/task/clientfilesystemwrite.go
View File

@ -21,7 +21,7 @@ func init() {
type clientFilesystemWriteTask struct { type clientFilesystemWriteTask struct {
} }
func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String() path, err := v.Lookup("path").String()
if err != nil { if err != nil {
return nil, err return nil, err
@ -39,7 +39,7 @@ func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Co
return compiler.NewValue(), nil return compiler.NewValue(), nil
} }
func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
contents := v.Lookup("contents") contents := v.Lookup("contents")
@ -79,7 +79,7 @@ func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plan
return fmt.Errorf("unsupported type %q", k) return fmt.Errorf("unsupported type %q", k)
} }
func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
contents, err := pctx.FS.FromValue(v) contents, err := pctx.FS.FromValue(v)
if err != nil { if err != nil {
return err return err

2
plan/task/clientnetwork.go
View File

@ -20,7 +20,7 @@ func init() {
type clientNetwork struct { type clientNetwork struct {
} }
func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
addr, err := v.Lookup("address").String() addr, err := v.Lookup("address").String()

2
plan/task/clientplatform.go
View File

@ -16,7 +16,7 @@ func init() {
type clientPlatformTask struct { type clientPlatformTask struct {
} }
func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t clientPlatformTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, _ *compiler.Value) (*compiler.Value, error) {
return compiler.NewValue().FillFields(map[string]interface{}{ return compiler.NewValue().FillFields(map[string]interface{}{
"os": runtime.GOOS, "os": runtime.GOOS,
"arch": runtime.GOARCH, "arch": runtime.GOARCH,

2
plan/task/copy.go
View File

@ -16,7 +16,7 @@ func init() {
type copyTask struct { type copyTask struct {
} }
func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var err error var err error
input, err := pctx.FS.FromValue(v.Lookup("input")) input, err := pctx.FS.FromValue(v.Lookup("input"))

2
plan/task/decodesecret.go
View File

@ -20,7 +20,7 @@ func init() {
type decodeSecretTask struct { type decodeSecretTask struct {
} }
func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
lg.Debug().Msg("decoding secret") lg.Debug().Msg("decoding secret")

2
plan/task/diff.go
View File

@ -16,7 +16,7 @@ func init() {
type diffTask struct { type diffTask struct {
} }
func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *diffTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lowerFS, err := pctx.FS.FromValue(v.Lookup("lower")) lowerFS, err := pctx.FS.FromValue(v.Lookup("lower"))
if err != nil { if err != nil {
return nil, err return nil, err

2
plan/task/dockerfile.go
View File

@ -28,7 +28,7 @@ func init() {
type dockerfileTask struct { type dockerfileTask struct {
} }
func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
auths, err := v.Lookup("auth").Fields() auths, err := v.Lookup("auth").Fields()
if err != nil { if err != nil {

8
plan/task/exec.go
View File

@ -20,7 +20,7 @@ func init() {
type execTask struct { type execTask struct {
} }
func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *execTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
// Get input state // Get input state
input, err := pctx.FS.FromValue(v.Lookup("input")) input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil { if err != nil {
@ -52,7 +52,7 @@ func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.S
}) })
} }
func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) { func (t *execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
opts := []llb.RunOption{} opts := []llb.RunOption{}
var cmd struct { var cmd struct {
Args []string Args []string
@ -141,7 +141,7 @@ func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]ll
return opts, nil return opts, nil
} }
func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) { func (t *execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
opts := []llb.RunOption{} opts := []llb.RunOption{}
fields, err := mounts.Fields() fields, err := mounts.Fields()
if err != nil { if err != nil {
@ -165,7 +165,7 @@ func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([
return opts, err return opts, err
} }
func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { func (t *execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
typ, err := mnt.Lookup("type").String() typ, err := mnt.Lookup("type").String()
if err != nil { if err != nil {
return nil, err return nil, err

4
plan/task/export.go
View File

@ -25,7 +25,7 @@ func init() {
type exportTask struct { type exportTask struct {
} }
func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { func (t exportTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
dir, err := os.MkdirTemp("", "dagger-export-*") dir, err := os.MkdirTemp("", "dagger-export-*")
if err != nil { if err != nil {
return err return err
@ -37,7 +37,7 @@ func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *co
return nil return nil
} }
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
dir := pctx.TempDirs.Get(v.Path().String()) dir := pctx.TempDirs.Get(v.Path().String())

2
plan/task/gitpull.go
View File

@ -19,7 +19,7 @@ func init() {
type gitPullTask struct { type gitPullTask struct {
} }
func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var gitPull struct { var gitPull struct {
Remote string Remote string
Ref string Ref string

2
plan/task/httpfetch.go
View File

@ -21,7 +21,7 @@ func init() {
type httpFetchTask struct { type httpFetchTask struct {
} }
func (c httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var httpFetch struct { var httpFetch struct {
Source string Source string
Checksum string Checksum string

2
plan/task/merge.go
View File

@ -16,7 +16,7 @@ func init() {
type mergeTask struct { type mergeTask struct {
} }
func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
inputs, err := v.Lookup("inputs").List() inputs, err := v.Lookup("inputs").List()
if err != nil { if err != nil {
return nil, err return nil, err

2
plan/task/mkdir.go
View File

@ -18,7 +18,7 @@ func init() {
type mkdirTask struct { type mkdirTask struct {
} }
func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String() path, err := v.Lookup("path").String()
if err != nil { if err != nil {
return nil, err return nil, err

2
plan/task/newsecret.go
View File

@ -18,7 +18,7 @@ func init() {
type newSecretTask struct { type newSecretTask struct {
} }
func (t *newSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *newSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String() path, err := v.Lookup("path").String()
if err != nil { if err != nil {
return nil, err return nil, err

2
plan/task/nop.go
View File

@ -15,6 +15,6 @@ func init() {
type nopTask struct { type nopTask struct {
} }
func (t *nopTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *nopTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
return v, nil return v, nil
} }

4
plan/task/pull.go
View File

@ -19,7 +19,7 @@ func init() {
type pullTask struct { type pullTask struct {
} }
func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
rawRef, err := v.Lookup("source").String() rawRef, err := v.Lookup("source").String()
@ -68,8 +68,8 @@ func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.
if err != nil { if err != nil {
return nil, err return nil, err
} }
fs := pctx.FS.New(result)
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{ return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(), "output": fs.MarshalCUE(),
"digest": digest, "digest": digest,

2
plan/task/push.go
View File

@ -19,7 +19,7 @@ func init() {
type pushTask struct { type pushTask struct {
} }
func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
rawDest, err := v.Lookup("dest").String() rawDest, err := v.Lookup("dest").String()

2
plan/task/readfile.go
View File

@ -18,7 +18,7 @@ func init() {
type readFileTask struct { type readFileTask struct {
} }
func (t *readFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *readFileTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String() path, err := v.Lookup("path").String()
if err != nil { if err != nil {
return nil, err return nil, err

4
plan/task/source.go
View File

@ -21,7 +21,7 @@ func init() {
type sourceTask struct { type sourceTask struct {
} }
func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { func (c *sourceTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
origPath, err := v.Lookup("path").String() origPath, err := v.Lookup("path").String()
if err != nil { if err != nil {
return err return err
@ -50,7 +50,7 @@ func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *c
return nil return nil
} }
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
path, err := v.Lookup("path").AbsPath() path, err := v.Lookup("path").AbsPath()

2
plan/task/task.go
View File

@ -40,7 +40,7 @@ const (
type NewFunc func() Task type NewFunc func() Task
type Task interface { type Task interface {
Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error)
} }
type PreRunner interface { type PreRunner interface {

2
plan/task/transformsecret.go
View File

@ -20,7 +20,7 @@ func init() {
type transformSecretTask struct { type transformSecretTask struct {
} }
func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx) lg := log.Ctx(ctx)
lg.Debug().Msg("transforming secret") lg.Debug().Msg("transforming secret")

2
plan/task/trimsecret.go
View File

@ -16,7 +16,7 @@ func init() {
type trimSecretTask struct { type trimSecretTask struct {
} }
func (t *trimSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *trimSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
input, err := pctx.Secrets.FromValue(v.Lookup("input")) input, err := pctx.Secrets.FromValue(v.Lookup("input"))
if err != nil { if err != nil {
return nil, err return nil, err

7
plan/task/writefile.go
View File

@ -19,7 +19,7 @@ func init() {
type writeFileTask struct { type writeFileTask struct {
} }
func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var contents []byte var contents []byte
var err error var err error
@ -49,19 +49,16 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
} }
permissions, err := v.Lookup("permissions").Int64() permissions, err := v.Lookup("permissions").Int64()
if err != nil { if err != nil {
return nil, err return nil, err
} }
input, err := pctx.FS.FromValue(v.Lookup("input")) input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil { if err != nil {
return nil, err return nil, err
} }
inputState, err := input.State() inputState, err := input.State()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -72,7 +69,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
) )
result, err := s.Solve(ctx, outputState, pctx.Platform.Get()) result, err := s.Solve(ctx, outputState, pctx.Platform.Get())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -80,7 +76,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
outputFS := pctx.FS.New(result) outputFS := pctx.FS.New(result)
output := compiler.NewValue() output := compiler.NewValue()
if err := output.FillPath(cue.ParsePath("output"), outputFS.MarshalCUE()); err != nil { if err := output.FillPath(cue.ParsePath("output"), outputFS.MarshalCUE()); err != nil {
return nil, err return nil, err
} }

47
solver/solver.go
View File

@ -25,6 +25,8 @@ type Solver struct {
opts Opts opts Opts
eventsWg *sync.WaitGroup eventsWg *sync.WaitGroup
closeCh chan *bk.SolveStatus closeCh chan *bk.SolveStatus
refs []bkgw.Reference
l sync.RWMutex
} }
type Opts struct { type Opts struct {
@ -34,10 +36,11 @@ type Opts struct {
Context *plancontext.Context Context *plancontext.Context
Auth *RegistryAuthProvider Auth *RegistryAuthProvider
NoCache bool NoCache bool
CacheImports []bkgw.CacheOptionsEntry
} }
func New(opts Opts) Solver { func New(opts Opts) *Solver {
return Solver{ return &Solver{
eventsWg: &sync.WaitGroup{}, eventsWg: &sync.WaitGroup{},
closeCh: make(chan *bk.SolveStatus), closeCh: make(chan *bk.SolveStatus),
opts: opts, opts: opts,
@ -63,25 +66,25 @@ func invalidateCache(def *llb.Definition) error {
return nil return nil
} }
func (s Solver) GetOptions() Opts { func (s *Solver) GetOptions() Opts {
return s.opts return s.opts
} }
func (s Solver) NoCache() bool { func (s *Solver) NoCache() bool {
return s.opts.NoCache return s.opts.NoCache
} }
func (s Solver) Stop() { func (s *Solver) Stop() {
close(s.closeCh) close(s.closeCh)
s.eventsWg.Wait() s.eventsWg.Wait()
close(s.opts.Events) close(s.opts.Events)
} }
func (s Solver) AddCredentials(target, username, secret string) { func (s *Solver) AddCredentials(target, username, secret string) {
s.opts.Auth.AddCredentials(target, username, secret) s.opts.Auth.AddCredentials(target, username, secret)
} }
func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) { func (s *Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) {
// FIXME: do not hardcode the platform // FIXME: do not hardcode the platform
def, err := st.Marshal(ctx, co...) def, err := st.Marshal(ctx, co...)
if err != nil { if err != nil {
@ -97,11 +100,11 @@ func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.Constraints
return def.ToPB(), nil return def.ToPB(), nil
} }
func (s Solver) SessionID() string { func (s *Solver) SessionID() string {
return s.opts.Gateway.BuildOpts().SessionID return s.opts.Gateway.BuildOpts().SessionID
} }
func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) { func (s *Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) {
var image dockerfile2llb.Image var image dockerfile2llb.Image
// Load image metadata and convert to to LLB. // Load image metadata and convert to to LLB.
@ -119,7 +122,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res
} }
// Solve will block until the state is solved and returns a Reference. // Solve will block until the state is solved and returns a Reference.
func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) { func (s *Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) {
// makes Solve() to block until LLB graph is solved. otherwise it will // makes Solve() to block until LLB graph is solved. otherwise it will
// return result (that you can for example use for next build) that // return result (that you can for example use for next build) that
// will be evaluated on export or if you access files on it. // will be evaluated on export or if you access files on it.
@ -131,9 +134,15 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.
return res, nil return res, nil
} }
func (s *Solver) References() []bkgw.Reference {
s.l.RLock()
defer s.l.RUnlock()
return s.refs
}
// Solve will block until the state is solved and returns a Reference. // Solve will block until the state is solved and returns a Reference.
// It takes a platform as argument which correspond to the targeted platform. // It takes a platform as argument which correspond to the targeted platform.
func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) { func (s *Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) {
def, err := s.Marshal(ctx, st, llb.Platform(platform)) def, err := s.Marshal(ctx, st, llb.Platform(platform))
if err != nil { if err != nil {
return nil, err return nil, err
@ -153,18 +162,28 @@ func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform
// call solve // call solve
res, err := s.SolveRequest(ctx, bkgw.SolveRequest{ res, err := s.SolveRequest(ctx, bkgw.SolveRequest{
Definition: def, Definition: def,
CacheImports: s.opts.CacheImports,
}) })
if err != nil { if err != nil {
return nil, err return nil, err
} }
return res.SingleRef() ref, err := res.SingleRef()
if err != nil {
return nil, err
}
s.l.Lock()
defer s.l.Unlock()
s.refs = append(s.refs, ref)
return ref, nil
} }
// Forward events from solver to the main events channel // Forward events from solver to the main events channel
// It creates a task in the solver waiting group to be // It creates a task in the solver waiting group to be
// sure that everything will be forward to the main channel // sure that everything will be forward to the main channel
func (s Solver) forwardEvents(ch chan *bk.SolveStatus) { func (s *Solver) forwardEvents(ch chan *bk.SolveStatus) {
s.eventsWg.Add(1) s.eventsWg.Add(1)
defer s.eventsWg.Done() defer s.eventsWg.Done()
@ -177,7 +196,7 @@ func (s Solver) forwardEvents(ch chan *bk.SolveStatus) {
// FIXME: this is currently implemented as a hack, starting a new Build session // FIXME: this is currently implemented as a hack, starting a new Build session
// within buildkit from the Control API. Ideally the Gateway API should allow to // within buildkit from the Control API. Ideally the Gateway API should allow to
// Export directly. // Export directly.
func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) { func (s *Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) {
// Check close event channel and return if we're already done with the main pipeline // Check close event channel and return if we're already done with the main pipeline
select { select {
case <-s.closeCh: case <-s.closeCh:

7
tests/helpers.bash
View File

@ -13,6 +13,13 @@ common_setup() {
DAGGER_TELEMETRY_DISABLE="1" DAGGER_TELEMETRY_DISABLE="1"
export DAGGER_TELEMETRY_DISABLE export DAGGER_TELEMETRY_DISABLE
export DAGGER_LOG_LEVEL="debug"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=integration-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=integration-tests-$BATS_TEST_NAME"
fi
SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt
export SOPS_AGE_KEY_FILE export SOPS_AGE_KEY_FILE
} }

2
tests/package.json
View File

@ -1,7 +1,7 @@
{ {
"license": "Apache-2.0", "license": "Apache-2.0",
"scripts": { "scripts": {
"test": "bats --jobs 4 --show-output-of-passing-tests --print-output-on-failure ." "test": "bats --jobs 4 --print-output-on-failure --verbose-run ."
}, },
"devDependencies": { "devDependencies": {
"bats": "https://github.com/bats-core/bats-core#master", "bats": "https://github.com/bats-core/bats-core#master",

2
tests/plan.bats
View File

@ -144,7 +144,7 @@ setup() {
cd "$TESTDIR/plan/client/filesystem/conflict" cd "$TESTDIR/plan/client/filesystem/conflict"
echo -n foo > test.txt echo -n foo > test.txt
run "$DAGGER" "do" --log-level debug -p . test run "$DAGGER" "do" -p . test
assert_line --regexp "client\.filesystem\..+\.write.+dependency=client\.filesystem\..+\.read" assert_line --regexp "client\.filesystem\..+\.write.+dependency=client\.filesystem\..+\.read"
rm -f test.txt rm -f test.txt