From 12be9a7dd815bba73c2a208e0360428e6323869e Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 15 Dec 2021 21:35:05 +0100 Subject: [PATCH] WIP: engine.#Exec Signed-off-by: Andrea Luzzardi --- docs/reference/europa/dagger/engine/README.md | 48 ++++ plan/task/exec.go | 264 ++++++++++++++++++ stdlib/europa/dagger/engine/exec.cue | 81 ++++++ tests/tasks.bats | 15 + tests/tasks/exec/args.cue | 26 ++ tests/tasks/exec/cue.mod/module.cue | 1 + tests/tasks/exec/cue.mod/pkg/.gitignore | 3 + tests/tasks/exec/env.cue | 24 ++ tests/tasks/exec/hosts.cue | 25 ++ tests/tasks/exec/mount_cache.cue | 63 +++++ tests/tasks/exec/mount_fs.cue | 37 +++ tests/tasks/exec/mount_secret.cue | 28 ++ tests/tasks/exec/mount_tmp.cue | 37 +++ tests/tasks/exec/user.cue | 41 +++ tests/tasks/exec/workdir.cue | 24 ++ 15 files changed, 717 insertions(+) create mode 100644 plan/task/exec.go create mode 100644 stdlib/europa/dagger/engine/exec.cue create mode 100644 tests/tasks/exec/args.cue create mode 100644 tests/tasks/exec/cue.mod/module.cue create mode 100644 tests/tasks/exec/cue.mod/pkg/.gitignore create mode 100644 tests/tasks/exec/env.cue create mode 100644 tests/tasks/exec/hosts.cue create mode 100644 tests/tasks/exec/mount_cache.cue create mode 100644 tests/tasks/exec/mount_fs.cue create mode 100644 tests/tasks/exec/mount_secret.cue create mode 100644 tests/tasks/exec/mount_tmp.cue create mode 100644 tests/tasks/exec/user.cue create mode 100644 tests/tasks/exec/workdir.cue diff --git a/docs/reference/europa/dagger/engine/README.md b/docs/reference/europa/dagger/engine/README.md index d3965862..efa36fc0 100644 --- a/docs/reference/europa/dagger/engine/README.md +++ b/docs/reference/europa/dagger/engine/README.md @@ -8,6 +8,18 @@ sidebar_label: engine import "alpha.dagger.io/europa/dagger/engine" ``` +## engine.#CacheDir + +A (best effort) persistent cache dir + +### engine.#CacheDir Inputs + +_No input._ + +### engine.#CacheDir Outputs + +_No output._ + ## engine.#Context ### engine.#Context Inputs @@ -18,6 +30,18 @@ _No input._ _No output._ +## engine.#Exec + +Execute a command in a container + +### engine.#Exec Inputs + +_No input._ + +### engine.#Exec Outputs + +_No output._ + ## engine.#FS A reference to a filesystem tree. For example: - The root filesystem of a container - A source code repository - A directory containing binary artifacts Rule of thumb: if it fits in a tar archive, it fits in a #FS. @@ -42,6 +66,18 @@ _No input._ _No output._ +## engine.#Mount + +A transient filesystem mount. + +### engine.#Mount Inputs + +_No input._ + +### engine.#Mount Outputs + +_No output._ + ## engine.#Plan A deployment plan executed by `dagger up` @@ -100,6 +136,18 @@ _No input._ _No output._ +## engine.#TempDir + +A temporary directory for command execution + +### engine.#TempDir Inputs + +_No input._ + +### engine.#TempDir Outputs + +_No output._ + ## engine.#WriteFile ### engine.#WriteFile Inputs diff --git a/plan/task/exec.go b/plan/task/exec.go new file mode 100644 index 00000000..a510fab6 --- /dev/null +++ b/plan/task/exec.go @@ -0,0 +1,264 @@ +package task + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/moby/buildkit/client/llb" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("Exec", func() Task { return &execTask{} }) +} + +type execTask struct { +} + +func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + // Get input state + input, err := pctx.FS.FromValue(v.Lookup("input")) + if err != nil { + return nil, err + } + st, err := input.Result().ToState() + if err != nil { + return nil, err + } + + // Run + opts, err := t.getRunOpts(v, pctx) + if err != nil { + return nil, err + } + st = st.Run(opts...).Root() + + // Solve + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + // Fill result + fs := pctx.FS.New(result) + return compiler.NewValue().FillFields(map[string]interface{}{ + "output": fs.MarshalCUE(), + "exit": 0, + }) +} + +func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) { + opts := []llb.RunOption{} + var cmd struct { + Args []string + Always bool + } + + if err := v.Decode(&cmd); err != nil { + return nil, err + } + // args + opts = append(opts, llb.Args(cmd.Args)) + + // workdir + workdir, err := v.Lookup("workdir").String() + if err != nil { + return nil, err + } + opts = append(opts, llb.Dir(workdir)) + + // env + envs, err := v.Lookup("env").Fields() + if err != nil { + return nil, err + } + for _, env := range envs { + v, err := env.Value.String() + if err != nil { + return nil, err + } + opts = append(opts, llb.AddEnv(env.Label(), v)) + } + + // always? + if cmd.Always { + // FIXME: also disables persistent cache directories + // There's an ongoing proposal that would fix this: https://github.com/moby/buildkit/issues/1213 + opts = append(opts, llb.IgnoreCache) + } + + hosts, err := v.Lookup("hosts").Fields() + if err != nil { + return nil, err + } + for _, host := range hosts { + s, err := host.Value.String() + if err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + opts = append(opts, llb.AddExtraHost(host.Label(), net.ParseIP(s))) + } + + user, err := v.Lookup("user").String() + if err != nil { + return nil, err + } + opts = append(opts, llb.User(user)) + + // mounts + mntOpts, err := t.mountAll(pctx, v.Lookup("mounts")) + if err != nil { + return nil, err + } + opts = append(opts, mntOpts...) + + // marker for status events + // FIXME + args := make([]string, 0, len(cmd.Args)) + for _, a := range cmd.Args { + args = append(args, fmt.Sprintf("%q", a)) + } + opts = append(opts, withCustomName(v, "Exec [%s]", strings.Join(args, ", "))) + + return opts, nil +} + +func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) { + opts := []llb.RunOption{} + fields, err := mounts.Fields() + if err != nil { + return nil, err + } + for _, mnt := range fields { + dest, err := mnt.Value.Lookup("dest").String() + if err != nil { + return nil, err + } + o, err := t.mount(pctx, dest, mnt.Value) + if err != nil { + return nil, err + } + opts = append(opts, o) + } + return opts, err +} + +func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { + typ, err := mnt.Lookup("type").String() + if err != nil { + return nil, err + } + switch typ { + case "cache": + return t.mountCache(pctx, dest, mnt) + case "tmp": + return t.mountTmp(pctx, dest, mnt) + case "service": + return t.mountService(pctx, dest, mnt) + case "fs": + return t.mountFS(pctx, dest, mnt) + case "secret": + return t.mountSecret(pctx, dest, mnt) + case "": + return nil, errors.New("no mount type specified") + default: + return nil, fmt.Errorf("unsupported mount type %q", typ) + } +} + +func (t *execTask) mountTmp(_ *plancontext.Context, dest string, _ *compiler.Value) (llb.RunOption, error) { + // FIXME: handle size + return llb.AddMount( + dest, + llb.Scratch(), + llb.Tmpfs(), + ), nil +} + +func (t *execTask) mountCache(_ *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { + contents := mnt.Lookup("contents") + id, err := contents.Lookup("id").String() + if err != nil { + return nil, err + } + + // FIXME: handle concurrency + concurrency := llb.CacheMountShared + + return llb.AddMount( + dest, + llb.Scratch(), + llb.AsPersistentCacheDir( + id, + concurrency, + ), + ), nil +} + +func (t *execTask) mountFS(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { + contents, err := pctx.FS.FromValue(mnt.Lookup("contents")) + if err != nil { + return nil, err + } + + // possibly construct mount options for LLB from + var mo []llb.MountOption + + // handle "path" option + if source := mnt.Lookup("source"); source.Exists() { + src, err := source.String() + if err != nil { + return nil, err + } + mo = append(mo, llb.SourcePath(src)) + } + + // FIXME: handle readonly + // if readonly := mnt.Lookup("ro"); readonly.Exists() { + // ro, err := readonly.Cue().Bool() + // if err != nil { + // return nil, err + // } + // } + + st, err := contents.Result().ToState() + if err != nil { + return nil, err + } + + return llb.AddMount(dest, st, mo...), nil +} + +func (t *execTask) mountSecret(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { + contents, err := pctx.Secrets.FromValue(mnt.Lookup("contents")) + if err != nil { + return nil, err + } + + // FIXME: handle uid, gid, optional + return llb.AddSecret(dest, + llb.SecretID(contents.ID()), + llb.SecretFileOpt(0, 0, 0400), // uid, gid, mask) + ), nil +} + +func (t *execTask) mountService(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) { + contents, err := pctx.Services.FromValue(mnt.Lookup("contents")) + if err != nil { + return nil, err + } + + return llb.AddSSHSocket( + llb.SSHID(contents.ID()), + llb.SSHSocketTarget(dest), + ), nil +} diff --git a/stdlib/europa/dagger/engine/exec.cue b/stdlib/europa/dagger/engine/exec.cue new file mode 100644 index 00000000..00152fe7 --- /dev/null +++ b/stdlib/europa/dagger/engine/exec.cue @@ -0,0 +1,81 @@ +package engine + +// Execute a command in a container +#Exec: { + _type: "Exec" + + // Container filesystem + input: #FS + + // Transient filesystem mounts + // Key is an arbitrary name, for example "app source code" + // Value is mount configuration + mounts: [name=string]: #Mount + + // Command to execute + // Example: ["echo", "hello, world!"] + args: [...string] + + // Environment variables + env: [key=string]: string + + // Working directory + workdir: string | *"/" + + // User ID or name + user: string | *"root" + + // If set, always execute even if the operation could be cached + always: true | *false + + // Inject hostname resolution into the container + // key is hostname, value is IP + hosts: [hostname=string]: string + + // Modified filesystem + output: #FS + + // Command exit code + // Currently this field can only ever be zero. + // If the command fails, DAG execution is immediately terminated. + // FIXME: expand API to allow custom handling of failed commands + exit: int & 0 +} + +// A transient filesystem mount. +#Mount: { + dest: string + type: string + { + type: "cache" + contents: #CacheDir + } | { + type: "tmp" + contents: #TempDir + } | { + type: "service" + contents: #Service + } | { + type: "fs" + contents: #FS + source?: string + ro?: true | *false + } | { + type: "secret" + contents: #Secret + uid: uint32 | *0 + gid: uint32 | *0 + optional: true | *false + } +} + +// A (best effort) persistent cache dir +#CacheDir: { + id: string + concurrency: *"shared" | "private" | "locked" +} + +// A temporary directory for command execution +#TempDir: { + size: int64 | *0 +} diff --git a/tests/tasks.bats b/tests/tasks.bats index 1eb058f0..88d2e163 100644 --- a/tests/tasks.bats +++ b/tests/tasks.bats @@ -23,4 +23,19 @@ setup() { cd "$TESTDIR"/tasks/writefile run "$DAGGER" --europa up ./writefile_failure_diff_contents.cue assert_failure +} + +@test "task: #Exec" { + cd "$TESTDIR"/tasks/exec + "$DAGGER" --europa up ./args.cue + "$DAGGER" --europa up ./env.cue + "$DAGGER" --europa up ./hosts.cue + + "$DAGGER" --europa up ./mount_cache.cue + "$DAGGER" --europa up ./mount_fs.cue + TESTSECRET="hello world" "$DAGGER" --europa up ./mount_secret.cue + "$DAGGER" --europa up ./mount_tmp.cue + + "$DAGGER" --europa up ./user.cue + "$DAGGER" --europa up ./workdir.cue } \ No newline at end of file diff --git a/tests/tasks/exec/args.cue b/tests/tasks/exec/args.cue new file mode 100644 index 00000000..398ca6b8 --- /dev/null +++ b/tests/tasks/exec/args.cue @@ -0,0 +1,26 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + exec: engine.#Exec & { + input: image.output + args: ["sh", "-c", "echo -n hello world > /output.txt"] + } + + verify: engine.#ReadFile & { + input: exec.output + path: "/output.txt" + } & { + // assert result + contents: "hello world" + } + } +} diff --git a/tests/tasks/exec/cue.mod/module.cue b/tests/tasks/exec/cue.mod/module.cue new file mode 100644 index 00000000..f8af9cef --- /dev/null +++ b/tests/tasks/exec/cue.mod/module.cue @@ -0,0 +1 @@ +module: "" diff --git a/tests/tasks/exec/cue.mod/pkg/.gitignore b/tests/tasks/exec/cue.mod/pkg/.gitignore new file mode 100644 index 00000000..2d4dc1ae --- /dev/null +++ b/tests/tasks/exec/cue.mod/pkg/.gitignore @@ -0,0 +1,3 @@ +# generated by dagger +alpha.dagger.io +dagger.lock diff --git a/tests/tasks/exec/env.cue b/tests/tasks/exec/env.cue new file mode 100644 index 00000000..0684057b --- /dev/null +++ b/tests/tasks/exec/env.cue @@ -0,0 +1,24 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + env: TEST: "hello world" + args: [ + "sh", "-c", + #""" + test "$TEST" = "hello world" + """#, + ] + } + } +} diff --git a/tests/tasks/exec/hosts.cue b/tests/tasks/exec/hosts.cue new file mode 100644 index 00000000..a5e3dc4f --- /dev/null +++ b/tests/tasks/exec/hosts.cue @@ -0,0 +1,25 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + hosts: "unit.test": "1.2.3.4" + args: [ + "sh", "-c", + #""" + grep -q "unit.test" /etc/hosts + grep -q "1.2.3.4" /etc/hosts + """#, + ] + } + } +} diff --git a/tests/tasks/exec/mount_cache.cue b/tests/tasks/exec/mount_cache.cue new file mode 100644 index 00000000..7688cde6 --- /dev/null +++ b/tests/tasks/exec/mount_cache.cue @@ -0,0 +1,63 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + sharedCache: engine.#CacheDir & { + id: "mycache" + } + + exec: engine.#Exec & { + input: image.output + mounts: cache: { + dest: "/cache" + contents: sharedCache + } + args: [ + "sh", "-c", + #""" + echo -n hello world > /cache/output.txt + """#, + ] + } + + verify: engine.#Exec & { + input: image.output + mounts: cache: { + dest: "/cache" + contents: sharedCache + } + args: [ + "sh", "-c", + #""" + test -f /cache/output.txt + test "$(cat /cache/output.txt)" = "hello world" + """#, + ] + } + + otherCache: engine.#CacheDir & { + id: "othercache" + } + verifyOtherCache: engine.#Exec & { + input: image.output + mounts: cache: { + dest: "/cache" + contents: otherCache + } + args: [ + "sh", "-c", + #""" + test ! -f /cache/output.txt + """#, + ] + } + } +} diff --git a/tests/tasks/exec/mount_fs.cue b/tests/tasks/exec/mount_fs.cue new file mode 100644 index 00000000..37d6d4dc --- /dev/null +++ b/tests/tasks/exec/mount_fs.cue @@ -0,0 +1,37 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + exec: engine.#Exec & { + input: image.output + args: [ + "sh", "-c", + #""" + echo -n hello world > /output.txt + """#, + ] + } + + verify: engine.#Exec & { + input: image.output + mounts: fs: { + dest: "/target" + contents: exec.output + } + args: [ + "sh", "-c", + #""" + test "$(cat /target/output.txt)" = "hello world" + """#, + ] + } + } +} diff --git a/tests/tasks/exec/mount_secret.cue b/tests/tasks/exec/mount_secret.cue new file mode 100644 index 00000000..34f9c7b3 --- /dev/null +++ b/tests/tasks/exec/mount_secret.cue @@ -0,0 +1,28 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + context: secrets: testSecret: envvar: "TESTSECRET" + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: context.secrets.testSecret.contents + } + args: [ + "sh", "-c", + #""" + test "$(cat /run/secrets/test)" = "hello world" + """#, + ] + } + } +} diff --git a/tests/tasks/exec/mount_tmp.cue b/tests/tasks/exec/mount_tmp.cue new file mode 100644 index 00000000..b3207144 --- /dev/null +++ b/tests/tasks/exec/mount_tmp.cue @@ -0,0 +1,37 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + exec: engine.#Exec & { + input: image.output + mounts: temp: { + dest: "/temp" + contents: engine.#TempDir + } + args: [ + "sh", "-c", + #""" + echo -n hello world > /temp/output.txt + """#, + ] + } + + verify: engine.#Exec & { + input: exec.output + args: [ + "sh", "-c", + #""" + test ! -f /temp/output.txt + """#, + ] + } + } +} diff --git a/tests/tasks/exec/user.cue b/tests/tasks/exec/user.cue new file mode 100644 index 00000000..55c98ced --- /dev/null +++ b/tests/tasks/exec/user.cue @@ -0,0 +1,41 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + addUser: engine.#Exec & { + input: image.output + args: ["adduser", "-D", "test"] + } + + verifyUsername: engine.#Exec & { + input: addUser.output + user: "test" + args: [ + "sh", "-c", + #""" + test "$(whoami)" = "test" + """#, + ] + } + + verifyUserID: engine.#Exec & { + input: addUser.output + user: "1000" + args: [ + "sh", "-c", + #""" + test "$(whoami)" = "test" + """#, + ] + } + + } +} diff --git a/tests/tasks/exec/workdir.cue b/tests/tasks/exec/workdir.cue new file mode 100644 index 00000000..1f005917 --- /dev/null +++ b/tests/tasks/exec/workdir.cue @@ -0,0 +1,24 @@ +package main + +import ( + "alpha.dagger.io/europa/dagger/engine" +) + +engine.#Plan & { + actions: { + image: engine.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + verify: engine.#Exec & { + input: image.output + workdir: "/tmp" + args: [ + "sh", "-c", + #""" + test "$(pwd)" = "/tmp" + """#, + ] + } + } +}