Merge pull request #1768 from sipsma/merge-diff

Add support for Merge and Diff fs operations.

pkg/dagger.io/dagger/fs.cue

@@ -88,11 +88,16 @@ package dagger
 
 // Merge multiple FS trees into one
 #Merge: {
-	@dagger(notimplemented)
 	$dagger: task: _name: "Merge"
+	inputs: [...#FS]
+	output: #FS
+}
 
-	input: #FS
-	layers: [...#CopyInfo]
+// Extract the difference from lower FS to upper FS as its own FS
+#Diff: {
+	$dagger: task: _name: "Diff"
+	lower:  #FS
+	upper:  #FS
 	output: #FS
 }
 

plan/task/diff.go

@@ -0,0 +1,48 @@
+package task
+
+import (
+	"context"
+
+	"github.com/moby/buildkit/client/llb"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Diff", func() Task { return &diffTask{} })
+}
+
+type diffTask struct {
+}
+
+func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	lowerFS, err := pctx.FS.FromValue(v.Lookup("lower"))
+	if err != nil {
+		return nil, err
+	}
+	lower, err := lowerFS.State()
+	if err != nil {
+		return nil, err
+	}
+
+	upperFS, err := pctx.FS.FromValue(v.Lookup("upper"))
+	if err != nil {
+		return nil, err
+	}
+	upper, err := upperFS.State()
+	if err != nil {
+		return nil, err
+	}
+
+	st := llb.Diff(lower, upper)
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": fs.MarshalCUE(),
+	})
+}

plan/task/merge.go

@@ -0,0 +1,48 @@
+package task
+
+import (
+	"context"
+
+	"github.com/moby/buildkit/client/llb"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Merge", func() Task { return &mergeTask{} })
+}
+
+type mergeTask struct {
+}
+
+func (t mergeTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	inputs, err := v.Lookup("inputs").List()
+	if err != nil {
+		return nil, err
+	}
+
+	inputStates := make([]llb.State, len(inputs))
+	for i, input := range inputs {
+		inputFS, err := pctx.FS.FromValue(input)
+		if err != nil {
+			return nil, err
+		}
+		inputState, err := inputFS.State()
+		if err != nil {
+			return nil, err
+		}
+		inputStates[i] = inputState
+	}
+
+	st := llb.Merge(inputStates)
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": fs.MarshalCUE(),
+	})
+}

tests/tasks.bats

@@ -128,3 +128,12 @@ setup() {
     run "$DAGGER" "do" -p ./tasks/source/source_not_exist.cue source
     assert_failure
 }
+
+@test "task: #Merge" {
+    "$DAGGER" "do" -p ./tasks/merge/merge.cue test
+}
+
+@test "task: #Diff" {
+    "$DAGGER" "do" -p ./tasks/diff/diff.cue test
+}
+

tests/tasks/diff/diff.cue

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"dagger.io/dagger"
+)
+
+dagger.#Plan & {
+	actions: {
+		alpineBase: dagger.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+		busyboxBase: dagger.#Pull & {
+			source: "busybox:1.34.1@sha256:1286c6d3c393023ef93c247724a6a2d665528144ffe07bacb741cc2b4edfefad"
+		}
+
+		exec1: dagger.#Exec & {
+			input: alpineBase.output
+			args: [
+				"sh", "-c",
+				#"""
+					mkdir /dir && echo -n foo > /dir/foo && echo -n removeme > /removeme
+					"""#,
+			]
+		}
+
+		exec2: dagger.#Exec & {
+			input: exec1.output
+			args: [
+				"sh", "-c",
+				#"""
+					echo -n bar > /dir/bar && rm removeme
+					"""#,
+			]
+		}
+
+		removeme: dagger.#WriteFile & {
+			input:    dagger.#Scratch
+			path:     "/removeme"
+			contents: "removeme"
+		}
+
+		test: {
+			diff: dagger.#Diff & {
+				lower: alpineBase.output
+				upper: exec2.output
+			}
+
+			verify_diff_foo: dagger.#ReadFile & {
+				input: diff.output
+				path:  "/dir/foo"
+			} & {
+				contents: "foo"
+			}
+			verify_diff_bar: dagger.#ReadFile & {
+				input: diff.output
+				path:  "/dir/bar"
+			} & {
+				contents: "bar"
+			}
+
+			mergediff: dagger.#Merge & {
+				inputs: [
+					busyboxBase.output,
+					removeme.output,
+					diff.output,
+				]
+			}
+			verify_remove: dagger.#Exec & {
+				input: mergediff.output
+				args: ["test", "!", "-e", "/removeme"]
+			}
+			verify_no_alpine_base: dagger.#Exec & {
+				input: mergediff.output
+				// make sure the the Diff actually separated files from the base
+				// by testing the non-existence of a file that only exists in the 
+				// alpine base, not busybox
+				args: ["test", "!", "-e", "/etc/alpine-release"]
+			}
+		}
+	}
+}

tests/tasks/merge/merge.cue

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"dagger.io/dagger"
+)
+
+dagger.#Plan & {
+	actions: {
+		image: dagger.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		exec: dagger.#Exec & {
+			input: image.output
+			args: [
+				"sh", "-c",
+				#"""
+					echo -n hello world > /output.txt
+					"""#,
+			]
+		}
+
+		dir: dagger.#Mkdir & {
+			input: dagger.#Scratch
+			path:  "/dir"
+		}
+
+		dirfoo: dagger.#WriteFile & {
+			input:    dir.output
+			path:     "/dir/foo"
+			contents: "foo"
+		}
+
+		dirfoo2: dagger.#WriteFile & {
+			input:    dir.output
+			path:     "/dir/foo"
+			contents: "foo2"
+		}
+
+		dirbar: dagger.#WriteFile & {
+			input:    dir.output
+			path:     "/dir/bar"
+			contents: "bar"
+		}
+
+		test: {
+			merge: dagger.#Merge & {
+				inputs: [
+					dir.output,
+					dirfoo.output,
+					dirbar.output,
+					exec.output,
+					dirfoo2.output,
+				]
+			}
+
+			verify_merge_output: dagger.#ReadFile & {
+				input: merge.output
+				path:  "/output.txt"
+			} & {
+				contents: "hello world"
+			}
+			verify_merge_dirbar: dagger.#ReadFile & {
+				input: merge.output
+				path:  "/dir/bar"
+			} & {
+				contents: "bar"
+			}
+			verify_merge_dirfoo: dagger.#ReadFile & {
+				input: merge.output
+				path:  "/dir/foo"
+			} & {
+				contents: "foo2"
+			}
+		}
+	}
+}