Merge pull request #1312 from aluzzardi/embedding-poc

pkg/dagger.io/dagger/engine/fs.cue

@@ -1,5 +1,20 @@
 package engine
 
+// Access the source directory for the current CUE package
+// This may safely be called from any package
+#Source: {
+	$dagger: task: _name: "Source"
+
+	// Relative path to source.
+	path: string
+	// Optionally exclude certain files
+	include: [...string]
+	// Optionall include certain files
+	exclude: [...string]
+
+	output: #FS
+}
+
 // Create one or multiple directory in a container
 #Mkdir: {
 	$dagger: task: _name: "Mkdir"

plan/task/source.go

@@ -0,0 +1,101 @@
+package task
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Source", func() Task { return &sourceTask{} })
+}
+
+type sourceTask struct {
+}
+
+func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
+	path, err := v.Lookup("path").String()
+	if err != nil {
+		return err
+	}
+
+	if !fs.ValidPath(path) {
+		return fmt.Errorf("invalid path %q", path)
+	}
+
+	absPath, err := v.Lookup("path").AbsPath()
+	if err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("path %q does not exist", path)
+	}
+
+	pctx.LocalDirs.Add(absPath)
+
+	return nil
+}
+
+func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	lg := log.Ctx(ctx)
+
+	path, err := v.Lookup("path").AbsPath()
+	if err != nil {
+		return nil, err
+	}
+
+	var source struct {
+		Include []string
+		Exclude []string
+	}
+
+	if err := v.Decode(&source); err != nil {
+		return nil, err
+	}
+
+	lg.Debug().Str("path", path).Msg("loading local directory")
+	opts := []llb.LocalOption{
+		withCustomName(v, "Embed %s", path),
+		llb.IncludePatterns(source.Include),
+		llb.ExcludePatterns(source.Exclude),
+		// Without hint, multiple `llb.Local` operations on the
+		// same path get a different digest.
+		llb.SessionID(s.SessionID()),
+		llb.SharedKeyHint(path),
+	}
+
+	// FIXME: Remove the `Copy` and use `Local` directly.
+	//
+	// Copy'ing is a costly operation which should be unnecessary.
+	// However, using llb.Local directly breaks caching sometimes for unknown reasons.
+	st := llb.Scratch().File(
+		llb.Copy(
+			llb.Local(
+				path,
+				opts...,
+			),
+			"/",
+			"/",
+		),
+		withCustomName(v, "Embed %s [copy]", path),
+	)
+
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": fs.MarshalCUE(),
+	})
+}

tests/tasks.bats

@@ -132,3 +132,15 @@ setup() {
 
     "$DAGGER" --europa up ./newsecret.cue
 }
+
+@test "task: #Source" {
+    cd "$TESTDIR"/tasks/source
+    "$DAGGER" --europa up ./source.cue
+    "$DAGGER" --europa up ./source_include_exclude.cue
+
+    run "$DAGGER" --europa up ./source_invalid_path.cue
+    assert_failure
+
+    run "$DAGGER" --europa up ./source_not_exist.cue
+    assert_failure
+}

tests/tasks/source/hello.txt

@@ -0,0 +1 @@
+hello

tests/tasks/source/source.cue

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"dagger.io/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		source: engine.#Source & {
+			path: "."
+		}
+
+		exec: engine.#Exec & {
+			input: image.output
+			mounts: code: {
+				dest:     "/src"
+				contents: source.output
+			}
+			args: ["/src/test.sh"]
+		}
+
+		verifyHello: engine.#ReadFile & {
+			input: source.output
+			path:  "/hello.txt"
+		} & {
+			// assert result
+			contents: "hello\n"
+		}
+
+		verifyWorld: engine.#ReadFile & {
+			input: source.output
+			path:  "/world.txt"
+		} & {
+			// assert result
+			contents: "world\n"
+		}
+	}
+}

tests/tasks/source/source_include_exclude.cue

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"dagger.io/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		sourceInclude: engine.#Source & {
+			path: "."
+			include: ["hello.txt"]
+		}
+
+		sourceExclude: engine.#Source & {
+			path: "."
+			exclude: ["hello.txt"]
+		}
+
+		test: engine.#Exec & {
+			input: image.output
+			mounts: {
+				include: {
+					dest:     "/include"
+					contents: sourceInclude.output
+				}
+				exclude: {
+					dest:     "/exclude"
+					contents: sourceExclude.output
+				}
+			}
+
+			args: ["sh", "-c",
+				#"""
+					test "$(find /include/ | wc -l)" -eq 1
+					test -f /include/hello.txt
+					test ! -f /exclude/hello.txt
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/source/source_invalid_path.cue

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"dagger.io/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		source: engine.#Source & {
+			path: ".."
+		}
+	}
+}

tests/tasks/source/source_not_exist.cue

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"dagger.io/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		source: engine.#Source & {
+			path: "not/exist"
+		}
+	}
+}

tests/tasks/source/test.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo -n hello world > /test.txt

tests/tasks/source/world.txt

@@ -0,0 +1 @@
+world