kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Fix export cache issue

Browse Source 
Resolve #1551 and #1020.
We are never returning the result of solved operations so Buildkit could not
cache the layer.
This commit implements a simple system to forward operations' result to the
main build to cache it.

Signed-off-by: Vasek - Tom C <tom.chauveau@epitech.eu>
This commit is contained in:
Tom Chauveau
2022-03-23 23:02:17 +01:00
committed by Vasek - Tom C
parent 64cdadb85e
commit 19c0f999f4
42 changed files with 144 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
plan/plan.go
View File

@@ -168,7 +168,7 @@ func (p *Plan) prepare(ctx context.Context) error {
}
// Do executes an action in the plan
func (p *Plan) Do(ctx context.Context, path cue.Path, s solver.Solver) error {
func (p *Plan) Do(ctx context.Context, path cue.Path, s *solver.Solver) error {
ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up")
defer span.End()

4
plan/runner.go
View File

@@ -22,13 +22,13 @@ import (
type Runner struct {
pctx *plancontext.Context
target cue.Path
s solver.Solver
s *solver.Solver
tasks sync.Map
mirror *compiler.Value
l sync.Mutex
}
func NewRunner(pctx *plancontext.Context, target cue.Path, s solver.Solver) *Runner {
func NewRunner(pctx *plancontext.Context, target cue.Path, s *solver.Solver) *Runner {
return &Runner{
pctx: pctx,
target: target,

2
plan/task/clientcommand.go
View File

@@ -23,7 +23,7 @@ func init() {
type clientCommandTask struct {
}
func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var opts struct {
Name string
Args []string

2
plan/task/clientenv.go
View File

@@ -19,7 +19,7 @@ func init() {
type clientEnvTask struct {
}
func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
log.Ctx(ctx).Debug().Msg("loading environment variables")
fields, err := v.Fields()

8
plan/task/clientfilesystemread.go
View File

@@ -21,7 +21,7 @@ func init() {
type clientFilesystemReadTask struct {
}
func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (t clientFilesystemReadTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
path, err := t.parsePath(v)
if err != nil {
return err
@@ -38,7 +38,7 @@ func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.
return nil
}
func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := t.parsePath(v)
if err != nil {
return nil, err
@@ -70,7 +70,7 @@ func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err
return
}
func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) {
func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (interface{}, error) {
lg := log.Ctx(ctx)
contents := v.Lookup("contents")
@@ -97,7 +97,7 @@ func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *planco
return nil, fmt.Errorf("unsupported type %q", k)
}
func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
var dir struct {
Include []string
Exclude []string

6
plan/task/clientfilesystemwrite.go
View File

@@ -21,7 +21,7 @@ func init() {
type clientFilesystemWriteTask struct {
}
func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err
@@ -39,7 +39,7 @@ func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Co
return compiler.NewValue(), nil
}
func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
lg := log.Ctx(ctx)
contents := v.Lookup("contents")
@@ -79,7 +79,7 @@ func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plan
return fmt.Errorf("unsupported type %q", k)
}
func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
contents, err := pctx.FS.FromValue(v)
if err != nil {
return err

2
plan/task/clientnetwork.go
View File

@@ -20,7 +20,7 @@ func init() {
type clientNetwork struct {
}
func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
addr, err := v.Lookup("address").String()

2
plan/task/clientplatform.go
View File

@@ -16,7 +16,7 @@ func init() {
type clientPlatformTask struct {
}
func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t clientPlatformTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, _ *compiler.Value) (*compiler.Value, error) {
return compiler.NewValue().FillFields(map[string]interface{}{
"os": runtime.GOOS,
"arch": runtime.GOARCH,

2
plan/task/copy.go
View File

@@ -16,7 +16,7 @@ func init() {
type copyTask struct {
}
func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var err error
input, err := pctx.FS.FromValue(v.Lookup("input"))

2
plan/task/decodesecret.go
View File

@@ -20,7 +20,7 @@ func init() {
type decodeSecretTask struct {
}
func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
lg.Debug().Msg("decoding secret")

2
plan/task/diff.go
View File

@@ -16,7 +16,7 @@ func init() {
type diffTask struct {
}
func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *diffTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lowerFS, err := pctx.FS.FromValue(v.Lookup("lower"))
if err != nil {
return nil, err

2
plan/task/dockerfile.go
View File

@@ -28,7 +28,7 @@ func init() {
type dockerfileTask struct {
}
func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
auths, err := v.Lookup("auth").Fields()
if err != nil {

8
plan/task/exec.go
View File

@@ -20,7 +20,7 @@ func init() {
type execTask struct {
}
func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *execTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
// Get input state
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
@@ -52,7 +52,7 @@ func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.S
})
}
func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
func (t *execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
var cmd struct {
Args []string
@@ -141,7 +141,7 @@ func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]ll
return opts, nil
}
func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
func (t *execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
fields, err := mounts.Fields()
if err != nil {
@@ -165,7 +165,7 @@ func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([
return opts, err
}
func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
func (t *execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
typ, err := mnt.Lookup("type").String()
if err != nil {
return nil, err

4
plan/task/export.go
View File

@@ -25,7 +25,7 @@ func init() {
type exportTask struct {
}
func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (t exportTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
dir, err := os.MkdirTemp("", "dagger-export-*")
if err != nil {
return err
@@ -37,7 +37,7 @@ func (t exportTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *co
return nil
}
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t exportTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
dir := pctx.TempDirs.Get(v.Path().String())

2
plan/task/gitpull.go
View File

@@ -19,7 +19,7 @@ func init() {
type gitPullTask struct {
}
func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var gitPull struct {
Remote string
Ref string

2
plan/task/httpfetch.go
View File

@@ -21,7 +21,7 @@ func init() {
type httpFetchTask struct {
}
func (c httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *httpFetchTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var httpFetch struct {
Source string
Checksum string

2
plan/task/merge.go
View File

@@ -16,7 +16,7 @@ func init() {
type mergeTask struct {
}
func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
inputs, err := v.Lookup("inputs").List()
if err != nil {
return nil, err

2
plan/task/mkdir.go
View File

@@ -18,7 +18,7 @@ func init() {
type mkdirTask struct {
}
func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *mkdirTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

2
plan/task/newsecret.go
View File

@@ -18,7 +18,7 @@ func init() {
type newSecretTask struct {
}
func (t *newSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *newSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

2
plan/task/nop.go
View File

@@ -15,6 +15,6 @@ func init() {
type nopTask struct {
}
func (t *nopTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *nopTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
return v, nil
}

4
plan/task/pull.go
View File

@@ -19,7 +19,7 @@ func init() {
type pullTask struct {
}
func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
rawRef, err := v.Lookup("source").String()
@@ -68,8 +68,8 @@ func (c *pullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.
if err != nil {
return nil, err
}
fs := pctx.FS.New(result)
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(),
"digest": digest,

2
plan/task/push.go
View File

@@ -19,7 +19,7 @@ func init() {
type pushTask struct {
}
func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *pushTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
rawDest, err := v.Lookup("dest").String()

2
plan/task/readfile.go
View File

@@ -18,7 +18,7 @@ func init() {
type readFileTask struct {
}
func (t *readFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *readFileTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
path, err := v.Lookup("path").String()
if err != nil {
return nil, err

4
plan/task/source.go
View File

@@ -21,7 +21,7 @@ func init() {
type sourceTask struct {
}
func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
func (c *sourceTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
origPath, err := v.Lookup("path").String()
if err != nil {
return err
@@ -50,7 +50,7 @@ func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *c
return nil
}
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
path, err := v.Lookup("path").AbsPath()

2
plan/task/task.go
View File

@@ -40,7 +40,7 @@ const (
type NewFunc func() Task
type Task interface {
Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error)
Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error)
}
type PreRunner interface {

2
plan/task/transformsecret.go
View File

@@ -20,7 +20,7 @@ func init() {
type transformSecretTask struct {
}
func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (c *transformSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
lg.Debug().Msg("transforming secret")

2
plan/task/trimsecret.go
View File

@@ -16,7 +16,7 @@ func init() {
type trimSecretTask struct {
}
func (t *trimSecretTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *trimSecretTask) Run(_ context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
input, err := pctx.Secrets.FromValue(v.Lookup("input"))
if err != nil {
return nil, err

7
plan/task/writefile.go
View File

@@ -19,7 +19,7 @@ func init() {
type writeFileTask struct {
}
func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
var contents []byte
var err error
@@ -49,19 +49,16 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
}
permissions, err := v.Lookup("permissions").Int64()
if err != nil {
return nil, err
}
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
return nil, err
}
inputState, err := input.State()
if err != nil {
return nil, err
}
@@ -72,7 +69,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
)
result, err := s.Solve(ctx, outputState, pctx.Platform.Get())
if err != nil {
return nil, err
}
@@ -80,7 +76,6 @@ func (t *writeFileTask) Run(ctx context.Context, pctx *plancontext.Context, s so
outputFS := pctx.FS.New(result)
output := compiler.NewValue()
if err := output.FillPath(cue.ParsePath("output"), outputFS.MarshalCUE()); err != nil {
return nil, err
}