kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Fix export cache issue

Browse Source 
Resolve #1551 and #1020.
We are never returning the result of solved operations so Buildkit could not
cache the layer.
This commit implements a simple system to forward operations' result to the
main build to cache it.

Signed-off-by: Vasek - Tom C <tom.chauveau@epitech.eu>
This commit is contained in:
Tom Chauveau 2022-03-23 23:02:17 +01:00 committed by Vasek - Tom C
parent 64cdadb85e
commit 19c0f999f4
No known key found for this signature in database
GPG Key ID: 175D82E572427960
42 changed files with 144 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/dagger-ci.yml vendored
View File

@ -2,7 +2,7 @@ name: "Dagger CI"
on:
push:
branches: [main]
branches: [ main ]
paths:
- '**.sh'
- '**.bash'
@ -14,7 +14,7 @@ on:
- 'go.sum'
- '.github/workflows/dagger-ci.yml'
pull_request:
branches: [main]
branches: [ main ]
paths:
- '**.sh'
- '**.bash'
@ -33,11 +33,10 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
-
name: Checkout
- name: Checkout
uses: actions/checkout@v2
-
name: Dagger CI
- name: Dagger CI
uses: dagger/dagger-for-github@v2
with:
workdir: ci

9
.github/workflows/test-integration.yml vendored
View File

@ -2,7 +2,7 @@ name: "Test Integration"
on:
push:
branches: [main]
branches: [ main ]
paths:
- "**.sh"
- "**.bash"
@ -16,7 +16,7 @@ on:
- "!docs/**"
pull_request:
branches: [main]
branches: [ main ]
paths:
- "**.sh"
- "**.bash"
@ -67,9 +67,6 @@ jobs:
- name: Test
env:
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
# TODO: https://github.com/dagger/dagger/pull/1341
# DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-integration"
# DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-integration"
run: |
env
make core-integration
make core-integration

3
.github/workflows/test-universe.yml vendored
View File

@ -57,8 +57,5 @@ jobs:
uses: crazy-max/ghaction-github-runtime@v1
- name: Test
env:
DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-universe"
DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-universe"
run: |
make universe-test

2
Makefile
View File

@ -93,4 +93,4 @@ web: # Run the website locally
.PHONY: todo
todo: # Find all TODO items
grep -r -A 1 "TODO:" $(CURDIR)
grep -r -A 1 "TODO:" $(CURDIR)

44
client/client.go
View File

@ -7,7 +7,7 @@ import (
"strings"
"sync"
"github.com/containerd/containerd/platforms"
"github.com/google/uuid"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
@ -18,7 +18,6 @@ import (
// buildkit
bk "github.com/moby/buildkit/client"
_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
@ -72,7 +71,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) {
}, nil
}
type DoFunc func(context.Context, solver.Solver) error
type DoFunc func(context.Context, *solver.Solver) error
// FIXME: return completed *Route, instead of *compiler.Value
func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error {
@ -96,6 +95,19 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) e
return eg.Wait()
}
func convertCacheOptionEntries(ims []bk.CacheOptionsEntry) []bkgw.CacheOptionsEntry {
convertIms := []bkgw.CacheOptionsEntry{}
for _, im := range ims {
convertIm := bkgw.CacheOptionsEntry{
Type: im.Type,
Attrs: im.Attrs,
}
convertIms = append(convertIms, convertIm)
}
return convertIms
}
func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error {
wg := sync.WaitGroup{}
@ -156,29 +168,31 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFu
resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
s := solver.New(solver.Opts{
Control: c.c,
Gateway: gw,
Events: eventsCh,
Auth: auth,
NoCache: c.cfg.NoCache,
Control: c.c,
Gateway: gw,
Events: eventsCh,
Auth: auth,
NoCache: c.cfg.NoCache,
CacheImports: convertCacheOptionEntries(opts.CacheImports),
})
// Close events channel
defer s.Stop()
// Compute output overlay
res := bkgw.NewResult()
if fn != nil {
if err := fn(ctx, s); err != nil {
err := fn(ctx, s)
if err != nil {
return nil, compiler.Err(err)
}
}
ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec())
if err != nil {
return nil, err
refs := s.References()
// Add functions layers
for _, ref := range refs {
res.AddRef(uuid.New().String(), ref)
}
}
res := bkgw.NewResult()
res.SetRef(ref)
return res, nil
}, buildCh)
if err != nil {

2
cmd/dagger/cmd/do.go
View File

@ -67,7 +67,7 @@ var doCmd = &cobra.Command{
Value: target.String(),
})
err = cl.Do(ctx, p.Context(), func(ctx context.Context, s solver.Solver) error {
err = cl.Do(ctx, p.Context(), func(ctx context.Context, s *solver.Solver) error {
return p.Do(ctx, target, s)
})

6
docs/learn/tests/helpers.bash
View File

@ -28,6 +28,12 @@ common_setup() {
export DAGGER_SANDBOX
dagger init --project "$DAGGER_SANDBOX"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=docs-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=docs-tests-$BATS_TEST_NAME"
fi
# allows the use of `sops`
SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt
export SOPS_AGE_KEY_FILE

7
pkg/universe.dagger.io/bats_helpers.bash
View File

@ -15,6 +15,13 @@ common_setup() {
DAGGER_LOG_FORMAT="plain"
export DAGGER_LOG_FORMAT
export DAGGER_LOG_LEVEL="debug"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=universe-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=universe-tests-$BATS_TEST_NAME"
fi
# cd into the directory containing the bats file
cd "$BATS_TEST_DIRNAME" || exit 1
}

2
pkg/universe.dagger.io/package.json
View File

@ -1,7 +1,7 @@
{
"license": "Apache-2.0",
"scripts": {
"test": "bats --report-formatter junit --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')"
"test": "bats --report-formatter junit --print-output-on-failure --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')"
},
"devDependencies": {
"bats": "^1.5.0",

2
plan/plan.go
View File

@ -168,7 +168,7 @@ func (p *Plan) prepare(ctx context.Context) error {
}
// Do executes an action in the plan
func (p *Plan) Do(ctx context.Context, path cue.Path, s solver.Solver) error {
func (p *Plan) Do(ctx context.Context, path cue.Path, s *solver.Solver) error {
ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up")
defer span.End()

4
plan/runner.go
View File

@ -22,13 +22,13 @@ import (
type Runner struct {
pctx *plancontext.Context
target cue.Path
s solver.Solver
s *solver.Solver
tasks sync.Map
mirror *compiler.Value
l sync.Mutex
}
func NewRunner(pctx *plancontext.Context, target cue.Path, s solver.Solver) *Runner {
func NewRunner(pctx *plancontext.Context, target cue.Path, s *solver.Solver) *Runner {
return &Runner{
pctx: pctx,
target: target,

2
plan/task/clientcommand.go
View File

@ -23,7 +23,7 @@ func init() {
type clientCommandTask struct {
}
func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var opts struct {
Name string
Args []string

2
plan/task/clientenv.go
View File

@ -19,7 +19,7 @@ func init() {
type clientEnvTask struct {
}
func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
log.Ctx(ctx).Debug().Msg("loading environment variables")
fields, err := v.Fields()

8
plan/task/clientfilesystemread.go
View File

@ -21,7 +21,7 @@ func init() {
type clientFilesystemReadTask struct {
}
func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (t clientFilesystemReadTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
path, err := t.parsePath(v)
if err != nil {
return err
@ -38,7 +38,7 @@ func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.
return nil
}
func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := t.parsePath(v)
if err != nil {
return nil, err
@ -70,7 +70,7 @@ func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err
return
}
func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) {
func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (interface{}, error) {
lg := log.Ctx(ctx)
contents := v.Lookup("contents")
@ -97,7 +97,7 @@ func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *planco
return nil, fmt.Errorf("unsupported type %q", k)
}
func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
var dir struct {
Include []string
Exclude []string

6
plan/task/clientfilesystemwrite.go
View File

@ -21,7 +21,7 @@ func init() {
type clientFilesystemWriteTask struct {
}
func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err
@ -39,7 +39,7 @@ func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Co
return compiler.NewValue(), nil
}
func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
lg := log.Ctx(ctx)
contents := v.Lookup("contents")
@ -79,7 +79,7 @@ func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plan
return fmt.Errorf("unsupported type %q", k)
}
func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
contents, err := pctx.FS.FromValue(v)
if err != nil {
return err

2
plan/task/clientnetwork.go
View File

@ -20,7 +20,7 @@ func init() {
type clientNetwork struct {
}
func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
addr, err := v.Lookup("address").String()

2
plan/task/clientplatform.go
View File

@ -16,7 +16,7 @@ func init() {
type clientPlatformTask struct {
}
func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientPlatformTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, _ *compiler.Value) (*compiler.Value, error) {
return compiler.NewValue().FillFields(map[string]interface{}{
"os": runtime.GOOS,
"arch": runtime.GOARCH,

2
plan/task/copy.go
View File

@ -16,7 +16,7 @@ func init() {
type copyTask struct {
}
func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var err error
input, err := pctx.FS.FromValue(v.Lookup("input"))

2
plan/task/decodesecret.go
View File

@ -20,7 +20,7 @@ func init() {
type decodeSecretTask struct {
}
func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
lg.Debug().Msg("decoding secret")

2
plan/task/diff.go
View File

@ -16,7 +16,7 @@ func init() {
type diffTask struct {
}
func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *diffTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lowerFS, err := pctx.FS.FromValue(v.Lookup("lower"))
if err != nil {
return nil, err

2
plan/task/dockerfile.go
View File

@ -28,7 +28,7 @@ func init() {
type dockerfileTask struct {
}
func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
auths, err := v.Lookup("auth").Fields()
if err != nil {

8
plan/task/exec.go
View File

@ -20,7 +20,7 @@ func init() {
type execTask struct {
}
func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *execTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
// Get input state
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
@ -52,7 +52,7 @@ func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.S
})
}
func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
func (t *execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
var cmd struct {
Args []string
@ -141,7 +141,7 @@ func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]ll
return opts, nil
}
func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
func (t *execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
fields, err := mounts.Fields()
if err != nil {
@ -165,7 +165,7 @@ func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([
return opts, err
}
func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
func (t *execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
typ, err := mnt.Lookup("type").String()
if err != nil {
return nil, err

4
plan/task/export.go
View File

@ -25,7 +25,7 @@ func init() {
type exportTask struct {
}
func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (t exportTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
dir, err := os.MkdirTemp("", "dagger-export-*")
if err != nil {
return err
@ -37,7 +37,7 @@ func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *co
return nil
}
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
dir := pctx.TempDirs.Get(v.Path().String())

2
plan/task/gitpull.go
View File

@ -19,7 +19,7 @@ func init() {
type gitPullTask struct {
}
func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var gitPull struct {
Remote string
Ref string

2
plan/task/httpfetch.go
View File

@ -21,7 +21,7 @@ func init() {
type httpFetchTask struct {
}
func (c httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var httpFetch struct {
Source string
Checksum string

2
plan/task/merge.go
View File

@ -16,7 +16,7 @@ func init() {
type mergeTask struct {
}
func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
inputs, err := v.Lookup("inputs").List()
if err != nil {
return nil, err

2
plan/task/mkdir.go
View File

@ -18,7 +18,7 @@ func init() {
type mkdirTask struct {
}
func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

2
plan/task/newsecret.go
View File

@ -18,7 +18,7 @@ func init() {
type newSecretTask struct {
}
func (t *newSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *newSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

2
plan/task/nop.go
View File

@ -15,6 +15,6 @@ func init() {
type nopTask struct {
}
func (t *nopTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *nopTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
return v, nil
}

4
plan/task/pull.go
View File

@ -19,7 +19,7 @@ func init() {
type pullTask struct {
}
func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
rawRef, err := v.Lookup("source").String()
@ -68,8 +68,8 @@ func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.
if err != nil {
return nil, err
}
fs := pctx.FS.New(result)
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(),
"digest": digest,

2
plan/task/push.go
View File

@ -19,7 +19,7 @@ func init() {
type pushTask struct {
}
func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
rawDest, err := v.Lookup("dest").String()

2
plan/task/readfile.go
View File

@ -18,7 +18,7 @@ func init() {
type readFileTask struct {
}
func (t *readFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *readFileTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

4
plan/task/source.go
View File

@ -21,7 +21,7 @@ func init() {
type sourceTask struct {
}
func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (c *sourceTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
origPath, err := v.Lookup("path").String()
if err != nil {
return err
@ -50,7 +50,7 @@ func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *c
return nil
}
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
path, err := v.Lookup("path").AbsPath()

2
plan/task/task.go
View File

@ -40,7 +40,7 @@ const (
type NewFunc func() Task
type Task interface {
Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error)
Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error)
}
type PreRunner interface {

2
plan/task/transformsecret.go
View File

@ -20,7 +20,7 @@ func init() {
type transformSecretTask struct {
}
func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
lg.Debug().Msg("transforming secret")

2
plan/task/trimsecret.go
View File

@ -16,7 +16,7 @@ func init() {
type trimSecretTask struct {
}
func (t *trimSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *trimSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
input, err := pctx.Secrets.FromValue(v.Lookup("input"))
if err != nil {
return nil, err

7
plan/task/writefile.go
View File

@ -19,7 +19,7 @@ func init() {
type writeFileTask struct {
}
func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var contents []byte
var err error
@ -49,19 +49,16 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
}
permissions, err := v.Lookup("permissions").Int64()
if err != nil {
return nil, err
}
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
return nil, err
}
inputState, err := input.State()
if err != nil {
return nil, err
}
@ -72,7 +69,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
)
result, err := s.Solve(ctx, outputState, pctx.Platform.Get())
if err != nil {
return nil, err
}
@ -80,7 +76,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
outputFS := pctx.FS.New(result)
output := compiler.NewValue()
if err := output.FillPath(cue.ParsePath("output"), outputFS.MarshalCUE()); err != nil {
return nil, err
}

61
solver/solver.go
View File

@ -25,19 +25,22 @@ type Solver struct {
opts Opts
eventsWg *sync.WaitGroup
closeCh chan *bk.SolveStatus
refs []bkgw.Reference
l sync.RWMutex
}
type Opts struct {
Control *bk.Client
Gateway bkgw.Client
Events chan *bk.SolveStatus
Context *plancontext.Context
Auth *RegistryAuthProvider
NoCache bool
Control *bk.Client
Gateway bkgw.Client
Events chan *bk.SolveStatus
Context *plancontext.Context
Auth *RegistryAuthProvider
NoCache bool
CacheImports []bkgw.CacheOptionsEntry
}
func New(opts Opts) Solver {
return Solver{
func New(opts Opts) *Solver {
return &Solver{
eventsWg: &sync.WaitGroup{},
closeCh: make(chan *bk.SolveStatus),
opts: opts,
@ -63,25 +66,25 @@ func invalidateCache(def *llb.Definition) error {
return nil
}
func (s Solver) GetOptions() Opts {
func (s *Solver) GetOptions() Opts {
return s.opts
}
func (s Solver) NoCache() bool {
func (s *Solver) NoCache() bool {
return s.opts.NoCache
}
func (s Solver) Stop() {
func (s *Solver) Stop() {
close(s.closeCh)
s.eventsWg.Wait()
close(s.opts.Events)
}
func (s Solver) AddCredentials(target, username, secret string) {
func (s *Solver) AddCredentials(target, username, secret string) {
s.opts.Auth.AddCredentials(target, username, secret)
}
func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) {
func (s *Solver) Marshal(ctx context.Context, st llb.State, co ...llb.ConstraintsOpt) (*bkpb.Definition, error) {
// FIXME: do not hardcode the platform
def, err := st.Marshal(ctx, co...)
if err != nil {
@ -97,11 +100,11 @@ func (s Solver) Marshal(ctx context.Context, st llb.State, co ...llb.Constraints
return def.ToPB(), nil
}
func (s Solver) SessionID() string {
func (s *Solver) SessionID() string {
return s.opts.Gateway.BuildOpts().SessionID
}
func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) {
func (s *Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, digest.Digest, error) {
var image dockerfile2llb.Image
// Load image metadata and convert to to LLB.
@ -119,7 +122,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res
}
// Solve will block until the state is solved and returns a Reference.
func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) {
func (s *Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) {
// makes Solve() to block until LLB graph is solved. otherwise it will
// return result (that you can for example use for next build) that
// will be evaluated on export or if you access files on it.
@ -131,9 +134,15 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.
return res, nil
}
func (s *Solver) References() []bkgw.Reference {
s.l.RLock()
defer s.l.RUnlock()
return s.refs
}
// Solve will block until the state is solved and returns a Reference.
// It takes a platform as argument which correspond to the targeted platform.
func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) {
func (s *Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform) (bkgw.Reference, error) {
def, err := s.Marshal(ctx, st, llb.Platform(platform))
if err != nil {
return nil, err
@ -152,19 +161,29 @@ func (s Solver) Solve(ctx context.Context, st llb.State, platform specs.Platform
// call solve
res, err := s.SolveRequest(ctx, bkgw.SolveRequest{
Definition: def,
Definition: def,
CacheImports: s.opts.CacheImports,
})
if err != nil {
return nil, err
}
return res.SingleRef()
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
s.l.Lock()
defer s.l.Unlock()
s.refs = append(s.refs, ref)
return ref, nil
}
// Forward events from solver to the main events channel
// It creates a task in the solver waiting group to be
// sure that everything will be forward to the main channel
func (s Solver) forwardEvents(ch chan *bk.SolveStatus) {
func (s *Solver) forwardEvents(ch chan *bk.SolveStatus) {
s.eventsWg.Add(1)
defer s.eventsWg.Done()
@ -177,7 +196,7 @@ func (s Solver) forwardEvents(ch chan *bk.SolveStatus) {
// FIXME: this is currently implemented as a hack, starting a new Build session
// within buildkit from the Control API. Ideally the Gateway API should allow to
// Export directly.
func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) {
func (s *Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Image, output bk.ExportEntry, platform specs.Platform) (*bk.SolveResponse, error) {
// Check close event channel and return if we're already done with the main pipeline
select {
case <-s.closeCh:

7
tests/helpers.bash
View File

@ -13,6 +13,13 @@ common_setup() {
DAGGER_TELEMETRY_DISABLE="1"
export DAGGER_TELEMETRY_DISABLE
export DAGGER_LOG_LEVEL="debug"
if [ -n "$GITHUB_ACTIONS" ];
then
export DAGGER_CACHE_TO="type=gha,mode=max,scope=integration-tests-$BATS_TEST_NAME"
export DAGGER_CACHE_FROM="type=gha,scope=integration-tests-$BATS_TEST_NAME"
fi
SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt
export SOPS_AGE_KEY_FILE
}

2
tests/package.json
View File

@ -1,7 +1,7 @@
{
"license": "Apache-2.0",
"scripts": {
"test": "bats --jobs 4 --show-output-of-passing-tests --print-output-on-failure ."
"test": "bats --jobs 4 --print-output-on-failure --verbose-run ."
},
"devDependencies": {
"bats": "https://github.com/bats-core/bats-core#master",

4
tests/plan.bats
View File

@ -144,7 +144,7 @@ setup() {
cd "$TESTDIR/plan/client/filesystem/conflict"
echo -n foo > test.txt
run "$DAGGER" "do" --log-level debug -p . test
run "$DAGGER" "do" -p . test
assert_line --regexp "client\.filesystem\..+\.write.+dependency=client\.filesystem\..+\.read"
rm -f test.txt
@ -221,4 +221,4 @@ setup() {
# Run with invalid platform
run "$DAGGER" "do" -p./plan/platform/config_platform_failure_invalid_platform.cue verify
assert_failure
}
}

2
tests/project.bats
View File

@ -25,6 +25,6 @@ setup() {
test -f ./cue.mod/pkg/.gitattributes
run cat ./cue.mod/pkg/.gitattributes
assert_output --partial "generated by dagger"
test ! -f ./cue.mod/pkg/.gitignore
}