From 6abe77729e4c0a31e719f2cd8df453bb5e4f2471 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 23 Dec 2021 20:23:52 +0100 Subject: [PATCH] europa: Code Embedding POC WIP PoC for code embedding Signed-off-by: Andrea Luzzardi --- pkg/dagger.io/dagger/engine/fs.cue | 15 +++ plan/task/source.go | 101 ++++++++++++++++++ tests/tasks.bats | 12 +++ tests/tasks/source/hello.txt | 1 + tests/tasks/source/source.cue | 42 ++++++++ tests/tasks/source/source_include_exclude.cue | 45 ++++++++ tests/tasks/source/source_invalid_path.cue | 17 +++ tests/tasks/source/source_not_exist.cue | 17 +++ tests/tasks/source/test.sh | 3 + tests/tasks/source/world.txt | 1 + 10 files changed, 254 insertions(+) create mode 100644 plan/task/source.go create mode 100644 tests/tasks/source/hello.txt create mode 100644 tests/tasks/source/source.cue create mode 100644 tests/tasks/source/source_include_exclude.cue create mode 100644 tests/tasks/source/source_invalid_path.cue create mode 100644 tests/tasks/source/source_not_exist.cue create mode 100755 tests/tasks/source/test.sh create mode 100644 tests/tasks/source/world.txt diff --git a/pkg/dagger.io/dagger/engine/fs.cue b/pkg/dagger.io/dagger/engine/fs.cue index 38589afd..c984b5bc 100644 --- a/pkg/dagger.io/dagger/engine/fs.cue +++ b/pkg/dagger.io/dagger/engine/fs.cue @@ -1,5 +1,20 @@ package engine +// Access the source directory for the current CUE package +// This may safely be called from any package +#Source: { + $dagger: task: _name: "Source" + + // Relative path to source. + path: string + // Optionally exclude certain files + include: [...string] + // Optionall include certain files + exclude: [...string] + + output: #FS +} + // Create one or multiple directory in a container #Mkdir: { $dagger: task: _name: "Mkdir" diff --git a/plan/task/source.go b/plan/task/source.go new file mode 100644 index 00000000..40701934 --- /dev/null +++ b/plan/task/source.go @@ -0,0 +1,101 @@ +package task + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + + "github.com/moby/buildkit/client/llb" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Source", func() Task { return &sourceTask{} }) +} + +type sourceTask struct { +} + +func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { + path, err := v.Lookup("path").String() + if err != nil { + return err + } + + if !fs.ValidPath(path) { + return fmt.Errorf("invalid path %q", path) + } + + absPath, err := v.Lookup("path").AbsPath() + if err != nil { + return err + } + + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("path %q does not exist", path) + } + + pctx.LocalDirs.Add(absPath) + + return nil +} + +func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + + path, err := v.Lookup("path").AbsPath() + if err != nil { + return nil, err + } + + var source struct { + Include []string + Exclude []string + } + + if err := v.Decode(&source); err != nil { + return nil, err + } + + lg.Debug().Str("path", path).Msg("loading local directory") + opts := []llb.LocalOption{ + withCustomName(v, "Embed %s", path), + llb.IncludePatterns(source.Include), + llb.ExcludePatterns(source.Exclude), + // Without hint, multiple `llb.Local` operations on the + // same path get a different digest. + llb.SessionID(s.SessionID()), + llb.SharedKeyHint(path), + } + + // FIXME: Remove the `Copy` and use `Local` directly. + // + // Copy'ing is a costly operation which should be unnecessary. + // However, using llb.Local directly breaks caching sometimes for unknown reasons. + st := llb.Scratch().File( + llb.Copy( + llb.Local( + path, + opts..., + ), + "/", + "/", + ), + withCustomName(v, "Embed %s [copy]", path), + ) + + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": fs.MarshalCUE(), + }) +} diff --git a/tests/tasks.bats b/tests/tasks.bats index 31b02f38..25cee42b 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -132,3 +132,15 @@ setup() { "$DAGGER" --europa up ./newsecret.cue } + +@test "task: #Source" { + cd "$TESTDIR"/tasks/source + "$DAGGER" --europa up ./source.cue + "$DAGGER" --europa up ./source_include_exclude.cue + + run "$DAGGER" --europa up ./source_invalid_path.cue + assert_failure + + run "$DAGGER" --europa up ./source_not_exist.cue + assert_failure +} diff --git a/tests/tasks/source/hello.txt b/tests/tasks/source/hello.txt new file mode 100644 index 00000000..ce013625 --- /dev/null +++ b/tests/tasks/source/hello.txt @@ -0,0 +1 @@ +hello diff --git a/tests/tasks/source/source.cue b/tests/tasks/source/source.cue new file mode 100644 index 00000000..4f554df6 --- /dev/null +++ b/tests/tasks/source/source.cue @@ -0,0 +1,42 @@ +package main + +import ( + "dagger.io/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + source: engine.#Source & { + path: "." + } + + exec: engine.#Exec & { + input: image.output + mounts: code: { + dest: "/src" + contents: source.output + } + args: ["/src/test.sh"] + } + + verifyHello: engine.#ReadFile & { + input: source.output + path: "/hello.txt" + } & { + // assert result + contents: "hello\n" + } + + verifyWorld: engine.#ReadFile & { + input: source.output + path: "/world.txt" + } & { + // assert result + contents: "world\n" + } + } +} diff --git a/tests/tasks/source/source_include_exclude.cue b/tests/tasks/source/source_include_exclude.cue new file mode 100644 index 00000000..38e696e6 --- /dev/null +++ b/tests/tasks/source/source_include_exclude.cue @@ -0,0 +1,45 @@ +package main + +import ( + "dagger.io/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + sourceInclude: engine.#Source & { + path: "." + include: ["hello.txt"] + } + + sourceExclude: engine.#Source & { + path: "." + exclude: ["hello.txt"] + } + + test: engine.#Exec & { + input: image.output + mounts: { + include: { + dest: "/include" + contents: sourceInclude.output + } + exclude: { + dest: "/exclude" + contents: sourceExclude.output + } + } + + args: ["sh", "-c", + #""" + test "$(find /include/ | wc -l)" -eq 1 + test -f /include/hello.txt + test ! -f /exclude/hello.txt + """#, + ] + } + } +} diff --git a/tests/tasks/source/source_invalid_path.cue b/tests/tasks/source/source_invalid_path.cue new file mode 100644 index 00000000..126d4a29 --- /dev/null +++ b/tests/tasks/source/source_invalid_path.cue @@ -0,0 +1,17 @@ +package main + +import ( + "dagger.io/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + source: engine.#Source & { + path: ".." + } + } +} diff --git a/tests/tasks/source/source_not_exist.cue b/tests/tasks/source/source_not_exist.cue new file mode 100644 index 00000000..4102d239 --- /dev/null +++ b/tests/tasks/source/source_not_exist.cue @@ -0,0 +1,17 @@ +package main + +import ( + "dagger.io/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + source: engine.#Source & { + path: "not/exist" + } + } +} diff --git a/tests/tasks/source/test.sh b/tests/tasks/source/test.sh new file mode 100755 index 00000000..2d3ac979 --- /dev/null +++ b/tests/tasks/source/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo -n hello world > /test.txt diff --git a/tests/tasks/source/world.txt b/tests/tasks/source/world.txt new file mode 100644 index 00000000..cc628ccd --- /dev/null +++ b/tests/tasks/source/world.txt @@ -0,0 +1 @@ +world