From b67d2d44ad64dddbc18e6928aa239c7fa8eb25c6 Mon Sep 17 00:00:00 2001 From: Erik Sipsma Date: Thu, 10 Mar 2022 11:37:25 -0800 Subject: [PATCH] Add support for Merge and Diff fs operations. Signed-off-by: Erik Sipsma --- pkg/dagger.io/dagger/fs.cue | 11 +++-- plan/task/diff.go | 48 ++++++++++++++++++++++ plan/task/merge.go | 48 ++++++++++++++++++++++ tests/tasks.bats | 9 +++++ tests/tasks/diff/diff.cue | 81 +++++++++++++++++++++++++++++++++++++ tests/tasks/merge/merge.cue | 77 +++++++++++++++++++++++++++++++++++ 6 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 plan/task/diff.go create mode 100644 plan/task/merge.go create mode 100644 tests/tasks/diff/diff.cue create mode 100644 tests/tasks/merge/merge.cue diff --git a/pkg/dagger.io/dagger/fs.cue b/pkg/dagger.io/dagger/fs.cue index e41e7541..7840681c 100644 --- a/pkg/dagger.io/dagger/fs.cue +++ b/pkg/dagger.io/dagger/fs.cue @@ -88,11 +88,16 @@ package dagger // Merge multiple FS trees into one #Merge: { - @dagger(notimplemented) $dagger: task: _name: "Merge" + inputs: [...#FS] + output: #FS +} - input: #FS - layers: [...#CopyInfo] +// Extract the difference from lower FS to upper FS as its own FS +#Diff: { + $dagger: task: _name: "Diff" + lower: #FS + upper: #FS output: #FS } diff --git a/plan/task/diff.go b/plan/task/diff.go new file mode 100644 index 00000000..5adbf709 --- /dev/null +++ b/plan/task/diff.go @@ -0,0 +1,48 @@ +package task + +import ( + "context" + + "github.com/moby/buildkit/client/llb" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Diff", func() Task { return &diffTask{} }) +} + +type diffTask struct { +} + +func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lowerFS, err := pctx.FS.FromValue(v.Lookup("lower")) + if err != nil { + return nil, err + } + lower, err := lowerFS.State() + if err != nil { + return nil, err + } + + upperFS, err := pctx.FS.FromValue(v.Lookup("upper")) + if err != nil { + return nil, err + } + upper, err := upperFS.State() + if err != nil { + return nil, err + } + + st := llb.Diff(lower, upper) + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": fs.MarshalCUE(), + }) +} diff --git a/plan/task/merge.go b/plan/task/merge.go new file mode 100644 index 00000000..1c5c18b1 --- /dev/null +++ b/plan/task/merge.go @@ -0,0 +1,48 @@ +package task + +import ( + "context" + + "github.com/moby/buildkit/client/llb" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Merge", func() Task { return &mergeTask{} }) +} + +type mergeTask struct { +} + +func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + inputs, err := v.Lookup("inputs").List() + if err != nil { + return nil, err + } + + inputStates := make([]llb.State, len(inputs)) + for i, input := range inputs { + inputFS, err := pctx.FS.FromValue(input) + if err != nil { + return nil, err + } + inputState, err := inputFS.State() + if err != nil { + return nil, err + } + inputStates[i] = inputState + } + + st := llb.Merge(inputStates) + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": fs.MarshalCUE(), + }) +} diff --git a/tests/tasks.bats b/tests/tasks.bats index 39340cb6..a93e1af2 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -128,3 +128,12 @@ setup() { run "$DAGGER" "do" -p ./tasks/source/source_not_exist.cue source assert_failure } + +@test "task: #Merge" { + "$DAGGER" "do" -p ./tasks/merge/merge.cue test +} + +@test "task: #Diff" { + "$DAGGER" "do" -p ./tasks/diff/diff.cue test +} + diff --git a/tests/tasks/diff/diff.cue b/tests/tasks/diff/diff.cue new file mode 100644 index 00000000..77574a08 --- /dev/null +++ b/tests/tasks/diff/diff.cue @@ -0,0 +1,81 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + actions: { + alpineBase: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + busyboxBase: dagger.#Pull & { + source: "busybox:1.34.1@sha256:1286c6d3c393023ef93c247724a6a2d665528144ffe07bacb741cc2b4edfefad" + } + + exec1: dagger.#Exec & { + input: alpineBase.output + args: [ + "sh", "-c", + #""" + mkdir /dir && echo -n foo > /dir/foo && echo -n removeme > /removeme + """#, + ] + } + + exec2: dagger.#Exec & { + input: exec1.output + args: [ + "sh", "-c", + #""" + echo -n bar > /dir/bar && rm removeme + """#, + ] + } + + removeme: dagger.#WriteFile & { + input: dagger.#Scratch + path: "/removeme" + contents: "removeme" + } + + test: { + diff: dagger.#Diff & { + lower: alpineBase.output + upper: exec2.output + } + + verify_diff_foo: dagger.#ReadFile & { + input: diff.output + path: "/dir/foo" + } & { + contents: "foo" + } + verify_diff_bar: dagger.#ReadFile & { + input: diff.output + path: "/dir/bar" + } & { + contents: "bar" + } + + mergediff: dagger.#Merge & { + inputs: [ + busyboxBase.output, + removeme.output, + diff.output, + ] + } + verify_remove: dagger.#Exec & { + input: mergediff.output + args: ["test", "!", "-e", "/removeme"] + } + verify_no_alpine_base: dagger.#Exec & { + input: mergediff.output + // make sure the the Diff actually separated files from the base + // by testing the non-existence of a file that only exists in the + // alpine base, not busybox + args: ["test", "!", "-e", "/etc/alpine-release"] + } + } + } +} diff --git a/tests/tasks/merge/merge.cue b/tests/tasks/merge/merge.cue new file mode 100644 index 00000000..d5ae7f2c --- /dev/null +++ b/tests/tasks/merge/merge.cue @@ -0,0 +1,77 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + exec: dagger.#Exec & { + input: image.output + args: [ + "sh", "-c", + #""" + echo -n hello world > /output.txt + """#, + ] + } + + dir: dagger.#Mkdir & { + input: dagger.#Scratch + path: "/dir" + } + + dirfoo: dagger.#WriteFile & { + input: dir.output + path: "/dir/foo" + contents: "foo" + } + + dirfoo2: dagger.#WriteFile & { + input: dir.output + path: "/dir/foo" + contents: "foo2" + } + + dirbar: dagger.#WriteFile & { + input: dir.output + path: "/dir/bar" + contents: "bar" + } + + test: { + merge: dagger.#Merge & { + inputs: [ + dir.output, + dirfoo.output, + dirbar.output, + exec.output, + dirfoo2.output, + ] + } + + verify_merge_output: dagger.#ReadFile & { + input: merge.output + path: "/output.txt" + } & { + contents: "hello world" + } + verify_merge_dirbar: dagger.#ReadFile & { + input: merge.output + path: "/dir/bar" + } & { + contents: "bar" + } + verify_merge_dirfoo: dagger.#ReadFile & { + input: merge.output + path: "/dir/foo" + } & { + contents: "foo2" + } + } + } +}