Merge pull request #1232 from aluzzardi/engine-exec

engine.#Exec

compiler/value.go

@@ -138,6 +138,11 @@ func (v *Value) Int64() (int64, error) {
 	return v.val.Int64()
 }
 
+// Proxy function to the underlying cue.Value
+func (v *Value) Bool() (bool, error) {
+	return v.val.Bool()
+}
+
 // Proxy function to the underlying cue.Value
 func (v *Value) Path() cue.Path {
 	return v.val.Path()

docs/reference/europa/dagger/engine/README.md

@@ -8,6 +8,18 @@ sidebar_label: engine
 import "alpha.dagger.io/europa/dagger/engine"
 ```
 
+## engine.#CacheDir
+
+A (best effort) persistent cache dir
+
+### engine.#CacheDir Inputs
+
+_No input._
+
+### engine.#CacheDir Outputs
+
+_No output._
+
 ## engine.#Context
 
 ### engine.#Context Inputs
@@ -18,6 +30,18 @@ _No input._
 
 _No output._
 
+## engine.#Exec
+
+Execute a command in a container
+
+### engine.#Exec Inputs
+
+_No input._
+
+### engine.#Exec Outputs
+
+_No output._
+
 ## engine.#FS
 
 A reference to a filesystem tree. For example:  - The root filesystem of a container  - A source code repository  - A directory containing binary artifacts Rule of thumb: if it fits in a tar archive, it fits in a #FS.
@@ -42,6 +66,18 @@ _No input._
 
 _No output._
 
+## engine.#Mount
+
+A transient filesystem mount.
+
+### engine.#Mount Inputs
+
+_No input._
+
+### engine.#Mount Outputs
+
+_No output._
+
 ## engine.#Plan
 
 A deployment plan executed by `dagger up`
@@ -100,6 +136,18 @@ _No input._
 
 _No output._
 
+## engine.#TempDir
+
+A temporary directory for command execution
+
+### engine.#TempDir Inputs
+
+_No input._
+
+### engine.#TempDir Outputs
+
+_No output._
+
 ## engine.#WriteFile
 
 ### engine.#WriteFile Inputs

plan/task/exec.go

@@ -0,0 +1,289 @@
+package task
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/moby/buildkit/client/llb"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("Exec", func() Task { return &execTask{} })
+}
+
+type execTask struct {
+}
+
+func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	// Get input state
+	input, err := pctx.FS.FromValue(v.Lookup("input"))
+	if err != nil {
+		return nil, err
+	}
+	st, err := input.Result().ToState()
+	if err != nil {
+		return nil, err
+	}
+
+	// Run
+	opts, err := t.getRunOpts(v, pctx)
+	if err != nil {
+		return nil, err
+	}
+	st = st.Run(opts...).Root()
+
+	// Solve
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	// Fill result
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": fs.MarshalCUE(),
+		"exit":   0,
+	})
+}
+
+func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
+	opts := []llb.RunOption{}
+	var cmd struct {
+		Args   []string
+		Always bool
+	}
+
+	if err := v.Decode(&cmd); err != nil {
+		return nil, err
+	}
+	// args
+	opts = append(opts, llb.Args(cmd.Args))
+
+	// workdir
+	workdir, err := v.Lookup("workdir").String()
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, llb.Dir(workdir))
+
+	// env
+	envs, err := v.Lookup("env").Fields()
+	if err != nil {
+		return nil, err
+	}
+	for _, env := range envs {
+		v, err := env.Value.String()
+		if err != nil {
+			return nil, err
+		}
+		opts = append(opts, llb.AddEnv(env.Label(), v))
+	}
+
+	// always?
+	if cmd.Always {
+		// FIXME: also disables persistent cache directories
+		// There's an ongoing proposal that would fix this: https://github.com/moby/buildkit/issues/1213
+		opts = append(opts, llb.IgnoreCache)
+	}
+
+	hosts, err := v.Lookup("hosts").Fields()
+	if err != nil {
+		return nil, err
+	}
+	for _, host := range hosts {
+		s, err := host.Value.String()
+		if err != nil {
+			return nil, err
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		opts = append(opts, llb.AddExtraHost(host.Label(), net.ParseIP(s)))
+	}
+
+	user, err := v.Lookup("user").String()
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, llb.User(user))
+
+	// mounts
+	mntOpts, err := t.mountAll(pctx, v.Lookup("mounts"))
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, mntOpts...)
+
+	// marker for status events
+	// FIXME
+	args := make([]string, 0, len(cmd.Args))
+	for _, a := range cmd.Args {
+		args = append(args, fmt.Sprintf("%q", a))
+	}
+	opts = append(opts, withCustomName(v, "Exec [%s]", strings.Join(args, ", ")))
+
+	return opts, nil
+}
+
+func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
+	opts := []llb.RunOption{}
+	fields, err := mounts.Fields()
+	if err != nil {
+		return nil, err
+	}
+	for _, mnt := range fields {
+		dest, err := mnt.Value.Lookup("dest").String()
+		if err != nil {
+			return nil, err
+		}
+		o, err := t.mount(pctx, dest, mnt.Value)
+		if err != nil {
+			return nil, err
+		}
+		opts = append(opts, o)
+	}
+	return opts, err
+}
+
+func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
+	typ, err := mnt.Lookup("type").String()
+	if err != nil {
+		return nil, err
+	}
+	switch typ {
+	case "cache":
+		return t.mountCache(pctx, dest, mnt)
+	case "tmp":
+		return t.mountTmp(pctx, dest, mnt)
+	case "service":
+		return t.mountService(pctx, dest, mnt)
+	case "fs":
+		return t.mountFS(pctx, dest, mnt)
+	case "secret":
+		return t.mountSecret(pctx, dest, mnt)
+	case "":
+		return nil, errors.New("no mount type specified")
+	default:
+		return nil, fmt.Errorf("unsupported mount type %q", typ)
+	}
+}
+
+func (t *execTask) mountTmp(_ *plancontext.Context, dest string, _ *compiler.Value) (llb.RunOption, error) {
+	// FIXME: handle size
+	return llb.AddMount(
+		dest,
+		llb.Scratch(),
+		llb.Tmpfs(),
+	), nil
+}
+
+func (t *execTask) mountCache(_ *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
+	contents := mnt.Lookup("contents")
+	id, err := contents.Lookup("id").String()
+	if err != nil {
+		return nil, err
+	}
+
+	concurrency, err := contents.Lookup("concurrency").String()
+	if err != nil {
+		return nil, err
+	}
+
+	var mode llb.CacheMountSharingMode
+	switch concurrency {
+	case "shared":
+		mode = llb.CacheMountShared
+	case "private":
+		mode = llb.CacheMountPrivate
+	case "locked":
+		mode = llb.CacheMountLocked
+	default:
+		return nil, fmt.Errorf("unknown concurrency mode %q", concurrency)
+	}
+
+	return llb.AddMount(
+		dest,
+		llb.Scratch(),
+		llb.AsPersistentCacheDir(
+			id,
+			mode,
+		),
+	), nil
+}
+
+func (t *execTask) mountFS(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
+	contents, err := pctx.FS.FromValue(mnt.Lookup("contents"))
+	if err != nil {
+		return nil, err
+	}
+
+	// possibly construct mount options for LLB from
+	var mo []llb.MountOption
+
+	// handle "path" option
+	if source := mnt.Lookup("source"); source.Exists() {
+		src, err := source.String()
+		if err != nil {
+			return nil, err
+		}
+		mo = append(mo, llb.SourcePath(src))
+	}
+
+	if ro := mnt.Lookup("ro"); ro.Exists() {
+		readonly, err := ro.Bool()
+		if err != nil {
+			return nil, err
+		}
+		if readonly {
+			mo = append(mo, llb.Readonly)
+		}
+	}
+
+	st, err := contents.Result().ToState()
+	if err != nil {
+		return nil, err
+	}
+
+	return llb.AddMount(dest, st, mo...), nil
+}
+
+func (t *execTask) mountSecret(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
+	contents, err := pctx.Secrets.FromValue(mnt.Lookup("contents"))
+	if err != nil {
+		return nil, err
+	}
+
+	opts := struct {
+		UID  int
+		GID  int
+		Mask int
+	}{}
+
+	if err := mnt.Decode(&opts); err != nil {
+		return nil, err
+	}
+
+	return llb.AddSecret(dest,
+		llb.SecretID(contents.ID()),
+		llb.SecretFileOpt(opts.UID, opts.GID, opts.Mask),
+	), nil
+}
+
+func (t *execTask) mountService(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
+	contents, err := pctx.Services.FromValue(mnt.Lookup("contents"))
+	if err != nil {
+		return nil, err
+	}
+
+	return llb.AddSSHSocket(
+		llb.SSHID(contents.ID()),
+		llb.SSHSocketTarget(dest),
+	), nil
+}

stdlib/europa/dagger/engine/exec.cue

@@ -0,0 +1,81 @@
+package engine
+
+// Execute a command in a container
+#Exec: {
+	_type: "Exec"
+
+	// Container filesystem
+	input: #FS
+
+	// Transient filesystem mounts
+	//   Key is an arbitrary name, for example "app source code"
+	//   Value is mount configuration
+	mounts: [name=string]: #Mount
+
+	// Command to execute
+	// Example: ["echo", "hello, world!"]
+	args: [...string]
+
+	// Environment variables
+	env: [key=string]: string
+
+	// Working directory
+	workdir: string | *"/"
+
+	// User ID or name
+	user: string | *"root"
+
+	// If set, always execute even if the operation could be cached
+	always: true | *false
+
+	// Inject hostname resolution into the container
+	// key is hostname, value is IP
+	hosts: [hostname=string]: string
+
+	// Modified filesystem
+	output: #FS
+
+	// Command exit code
+	// Currently this field can only ever be zero.
+	// If the command fails, DAG execution is immediately terminated.
+	// FIXME: expand API to allow custom handling of failed commands
+	exit: int & 0
+}
+
+// A transient filesystem mount.
+#Mount: {
+	dest: string
+	type: string
+	{
+		type:     "cache"
+		contents: #CacheDir
+	} | {
+		type:     "tmp"
+		contents: #TempDir
+	} | {
+		type:     "service"
+		contents: #Service
+	} | {
+		type:     "fs"
+		contents: #FS
+		source?:  string
+		ro?:      true | *false
+	} | {
+		type:     "secret"
+		contents: #Secret
+		uid:      int | *0
+		gid:      int | *0
+		mask:     int | *0o400
+	}
+}
+
+// A (best effort) persistent cache dir
+#CacheDir: {
+	id:          string
+	concurrency: *"shared" | "private" | "locked"
+}
+
+// A temporary directory for command execution
+#TempDir: {
+	size: int64 | *0
+}

tests/tasks.bats

@@ -23,4 +23,20 @@ setup() {
     cd "$TESTDIR"/tasks/writefile
     run "$DAGGER" --europa up ./writefile_failure_diff_contents.cue
     assert_failure 
+}
+
+@test "task: #Exec" {
+    cd "$TESTDIR"/tasks/exec
+    "$DAGGER" --europa up ./args.cue
+    "$DAGGER" --europa up ./env.cue
+    "$DAGGER" --europa up ./hosts.cue
+
+    "$DAGGER" --europa up ./mount_cache.cue
+    "$DAGGER" --europa up ./mount_fs.cue
+    TESTSECRET="hello world" "$DAGGER" --europa up ./mount_secret.cue
+    "$DAGGER" --europa up ./mount_tmp.cue
+    "$DAGGER" --europa up ./mount_service.cue
+
+    "$DAGGER" --europa up ./user.cue
+    "$DAGGER" --europa up ./workdir.cue
 }

tests/tasks/exec/args.cue

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		exec: engine.#Exec & {
+			input: image.output
+			args: ["sh", "-c", "echo -n hello world > /output.txt"]
+		}
+
+		verify: engine.#ReadFile & {
+			input: exec.output
+			path:  "/output.txt"
+		} & {
+			// assert result
+			contents: "hello world"
+		}
+	}
+}

tests/tasks/exec/cue.mod/module.cue

@@ -0,0 +1 @@
+module: ""

tests/tasks/exec/cue.mod/pkg/.gitignore

@@ -0,0 +1,3 @@
+# generated by dagger
+alpha.dagger.io
+dagger.lock

tests/tasks/exec/env.cue

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			env: TEST: "hello world"
+			args: [
+				"sh", "-c",
+				#"""
+					test "$TEST" = "hello world"
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/exec/hosts.cue

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			hosts: "unit.test": "1.2.3.4"
+			args: [
+				"sh", "-c",
+				#"""
+					grep -q "unit.test" /etc/hosts
+					grep -q "1.2.3.4" /etc/hosts
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/exec/mount_cache.cue

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		sharedCache: engine.#CacheDir & {
+			id: "mycache"
+		}
+
+		exec: engine.#Exec & {
+			input: image.output
+			mounts: cache: {
+				dest:     "/cache"
+				contents: sharedCache
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					echo -n hello world > /cache/output.txt
+					"""#,
+			]
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			mounts: cache: {
+				dest:     "/cache"
+				contents: sharedCache
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test -f /cache/output.txt
+					test "$(cat /cache/output.txt)" = "hello world"
+					"""#,
+			]
+		}
+
+		otherCache: engine.#CacheDir & {
+			id: "othercache"
+		}
+		verifyOtherCache: engine.#Exec & {
+			input: image.output
+			mounts: cache: {
+				dest:     "/cache"
+				contents: otherCache
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test ! -f /cache/output.txt
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/exec/mount_fs.cue

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		exec: engine.#Exec & {
+			input: image.output
+			args: [
+				"sh", "-c",
+				#"""
+					echo -n hello world > /output.txt
+					"""#,
+			]
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			mounts: fs: {
+				dest:     "/target"
+				contents: exec.output
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(cat /target/output.txt)" = "hello world"
+					touch /target/rw
+					"""#,
+			]
+		}
+
+		verifyRO: engine.#Exec & {
+			input: image.output
+			mounts: fs: {
+				dest:     "/target"
+				contents: exec.output
+				ro:       true
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(cat /target/output.txt)" = "hello world"
+
+					touch /target/ro && exit 1
+					true
+					"""#,
+			]
+		}
+
+		verifySource: engine.#Exec & {
+			input: image.output
+			mounts: fs: {
+				dest:     "/target.txt"
+				contents: exec.output
+				source:   "/output.txt"
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(cat /target.txt)" = "hello world"
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/exec/mount_secret.cue

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	context: secrets: testSecret: envvar: "TESTSECRET"
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			mounts: secret: {
+				dest:     "/run/secrets/test"
+				contents: context.secrets.testSecret.contents
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(cat /run/secrets/test)" = "hello world"
+					ls -l /run/secrets/test | grep -- "-r--------"
+					"""#,
+			]
+		}
+
+		verifyPerm: engine.#Exec & {
+			input: image.output
+			mounts: secret: {
+				dest:     "/run/secrets/test"
+				contents: context.secrets.testSecret.contents
+				uid:      42
+				gid:      24
+				mask:     0o666
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					ls -l /run/secrets/test | grep -- "-rw-rw-rw-"
+					ls -l /run/secrets/test | grep -- "42"
+					ls -l /run/secrets/test | grep -- "24"
+					"""#,
+			]
+		}
+
+	}
+}

tests/tasks/exec/mount_service.cue

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	context: services: dockerSocket: unix: "/var/run/docker.sock"
+
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		imageWithDocker: engine.#Exec & {
+			input: image.output
+			args: ["apk", "add", "--no-cache", "docker-cli"]
+		}
+
+		verify: engine.#Exec & {
+			input: imageWithDocker.output
+			mounts: docker: {
+				dest:     "/var/run/docker.sock"
+				contents: context.services.dockerSocket.service
+			}
+			args: ["docker", "info"]
+		}
+	}
+}

tests/tasks/exec/mount_tmp.cue

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		exec: engine.#Exec & {
+			input: image.output
+			mounts: temp: {
+				dest:     "/temp"
+				contents: engine.#TempDir
+			}
+			args: [
+				"sh", "-c",
+				#"""
+					echo -n hello world > /temp/output.txt
+					"""#,
+			]
+		}
+
+		verify: engine.#Exec & {
+			input: exec.output
+			args: [
+				"sh", "-c",
+				#"""
+					test ! -f /temp/output.txt
+					"""#,
+			]
+		}
+	}
+}

tests/tasks/exec/user.cue

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		addUser: engine.#Exec & {
+			input: image.output
+			args: ["adduser", "-D", "test"]
+		}
+
+		verifyUsername: engine.#Exec & {
+			input: addUser.output
+			user:  "test"
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(whoami)" = "test"
+					"""#,
+			]
+		}
+
+		verifyUserID: engine.#Exec & {
+			input: addUser.output
+			user:  "1000"
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(whoami)" = "test"
+					"""#,
+			]
+		}
+
+	}
+}

tests/tasks/exec/workdir.cue

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"alpha.dagger.io/europa/dagger/engine"
+)
+
+engine.#Plan & {
+	actions: {
+		image: engine.#Pull & {
+			source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
+		}
+
+		verify: engine.#Exec & {
+			input:   image.output
+			workdir: "/tmp"
+			args: [
+				"sh", "-c",
+				#"""
+					test "$(pwd)" = "/tmp"
+					"""#,
+			]
+		}
+	}
+}