Merge pull request #1232 from aluzzardi/engine-exec

engine.#Exec
This commit is contained in:
Joel Longtine 2021-12-17 11:03:58 -07:00 committed by GitHub
commit a9275d889f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 832 additions and 0 deletions

View File

@ -138,6 +138,11 @@ func (v *Value) Int64() (int64, error) {
return v.val.Int64() return v.val.Int64()
} }
// Proxy function to the underlying cue.Value
func (v *Value) Bool() (bool, error) {
return v.val.Bool()
}
// Proxy function to the underlying cue.Value // Proxy function to the underlying cue.Value
func (v *Value) Path() cue.Path { func (v *Value) Path() cue.Path {
return v.val.Path() return v.val.Path()

View File

@ -8,6 +8,18 @@ sidebar_label: engine
import "alpha.dagger.io/europa/dagger/engine" import "alpha.dagger.io/europa/dagger/engine"
``` ```
## engine.#CacheDir
A (best effort) persistent cache dir
### engine.#CacheDir Inputs
_No input._
### engine.#CacheDir Outputs
_No output._
## engine.#Context ## engine.#Context
### engine.#Context Inputs ### engine.#Context Inputs
@ -18,6 +30,18 @@ _No input._
_No output._ _No output._
## engine.#Exec
Execute a command in a container
### engine.#Exec Inputs
_No input._
### engine.#Exec Outputs
_No output._
## engine.#FS ## engine.#FS
A reference to a filesystem tree. For example: - The root filesystem of a container - A source code repository - A directory containing binary artifacts Rule of thumb: if it fits in a tar archive, it fits in a #FS. A reference to a filesystem tree. For example: - The root filesystem of a container - A source code repository - A directory containing binary artifacts Rule of thumb: if it fits in a tar archive, it fits in a #FS.
@ -42,6 +66,18 @@ _No input._
_No output._ _No output._
## engine.#Mount
A transient filesystem mount.
### engine.#Mount Inputs
_No input._
### engine.#Mount Outputs
_No output._
## engine.#Plan ## engine.#Plan
A deployment plan executed by `dagger up` A deployment plan executed by `dagger up`
@ -100,6 +136,18 @@ _No input._
_No output._ _No output._
## engine.#TempDir
A temporary directory for command execution
### engine.#TempDir Inputs
_No input._
### engine.#TempDir Outputs
_No output._
## engine.#WriteFile ## engine.#WriteFile
### engine.#WriteFile Inputs ### engine.#WriteFile Inputs

289
plan/task/exec.go Normal file
View File

@ -0,0 +1,289 @@
package task
import (
"context"
"errors"
"fmt"
"net"
"strings"
"github.com/moby/buildkit/client/llb"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
)
func init() {
Register("Exec", func() Task { return &execTask{} })
}
type execTask struct {
}
func (t execTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
// Get input state
input, err := pctx.FS.FromValue(v.Lookup("input"))
if err != nil {
return nil, err
}
st, err := input.Result().ToState()
if err != nil {
return nil, err
}
// Run
opts, err := t.getRunOpts(v, pctx)
if err != nil {
return nil, err
}
st = st.Run(opts...).Root()
// Solve
result, err := s.Solve(ctx, st, pctx.Platform.Get())
if err != nil {
return nil, err
}
// Fill result
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(),
"exit": 0,
})
}
func (t execTask) getRunOpts(v *compiler.Value, pctx *plancontext.Context) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
var cmd struct {
Args []string
Always bool
}
if err := v.Decode(&cmd); err != nil {
return nil, err
}
// args
opts = append(opts, llb.Args(cmd.Args))
// workdir
workdir, err := v.Lookup("workdir").String()
if err != nil {
return nil, err
}
opts = append(opts, llb.Dir(workdir))
// env
envs, err := v.Lookup("env").Fields()
if err != nil {
return nil, err
}
for _, env := range envs {
v, err := env.Value.String()
if err != nil {
return nil, err
}
opts = append(opts, llb.AddEnv(env.Label(), v))
}
// always?
if cmd.Always {
// FIXME: also disables persistent cache directories
// There's an ongoing proposal that would fix this: https://github.com/moby/buildkit/issues/1213
opts = append(opts, llb.IgnoreCache)
}
hosts, err := v.Lookup("hosts").Fields()
if err != nil {
return nil, err
}
for _, host := range hosts {
s, err := host.Value.String()
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
opts = append(opts, llb.AddExtraHost(host.Label(), net.ParseIP(s)))
}
user, err := v.Lookup("user").String()
if err != nil {
return nil, err
}
opts = append(opts, llb.User(user))
// mounts
mntOpts, err := t.mountAll(pctx, v.Lookup("mounts"))
if err != nil {
return nil, err
}
opts = append(opts, mntOpts...)
// marker for status events
// FIXME
args := make([]string, 0, len(cmd.Args))
for _, a := range cmd.Args {
args = append(args, fmt.Sprintf("%q", a))
}
opts = append(opts, withCustomName(v, "Exec [%s]", strings.Join(args, ", ")))
return opts, nil
}
func (t execTask) mountAll(pctx *plancontext.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
opts := []llb.RunOption{}
fields, err := mounts.Fields()
if err != nil {
return nil, err
}
for _, mnt := range fields {
dest, err := mnt.Value.Lookup("dest").String()
if err != nil {
return nil, err
}
o, err := t.mount(pctx, dest, mnt.Value)
if err != nil {
return nil, err
}
opts = append(opts, o)
}
return opts, err
}
func (t execTask) mount(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
typ, err := mnt.Lookup("type").String()
if err != nil {
return nil, err
}
switch typ {
case "cache":
return t.mountCache(pctx, dest, mnt)
case "tmp":
return t.mountTmp(pctx, dest, mnt)
case "service":
return t.mountService(pctx, dest, mnt)
case "fs":
return t.mountFS(pctx, dest, mnt)
case "secret":
return t.mountSecret(pctx, dest, mnt)
case "":
return nil, errors.New("no mount type specified")
default:
return nil, fmt.Errorf("unsupported mount type %q", typ)
}
}
func (t *execTask) mountTmp(_ *plancontext.Context, dest string, _ *compiler.Value) (llb.RunOption, error) {
// FIXME: handle size
return llb.AddMount(
dest,
llb.Scratch(),
llb.Tmpfs(),
), nil
}
func (t *execTask) mountCache(_ *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
contents := mnt.Lookup("contents")
id, err := contents.Lookup("id").String()
if err != nil {
return nil, err
}
concurrency, err := contents.Lookup("concurrency").String()
if err != nil {
return nil, err
}
var mode llb.CacheMountSharingMode
switch concurrency {
case "shared":
mode = llb.CacheMountShared
case "private":
mode = llb.CacheMountPrivate
case "locked":
mode = llb.CacheMountLocked
default:
return nil, fmt.Errorf("unknown concurrency mode %q", concurrency)
}
return llb.AddMount(
dest,
llb.Scratch(),
llb.AsPersistentCacheDir(
id,
mode,
),
), nil
}
func (t *execTask) mountFS(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
contents, err := pctx.FS.FromValue(mnt.Lookup("contents"))
if err != nil {
return nil, err
}
// possibly construct mount options for LLB from
var mo []llb.MountOption
// handle "path" option
if source := mnt.Lookup("source"); source.Exists() {
src, err := source.String()
if err != nil {
return nil, err
}
mo = append(mo, llb.SourcePath(src))
}
if ro := mnt.Lookup("ro"); ro.Exists() {
readonly, err := ro.Bool()
if err != nil {
return nil, err
}
if readonly {
mo = append(mo, llb.Readonly)
}
}
st, err := contents.Result().ToState()
if err != nil {
return nil, err
}
return llb.AddMount(dest, st, mo...), nil
}
func (t *execTask) mountSecret(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
contents, err := pctx.Secrets.FromValue(mnt.Lookup("contents"))
if err != nil {
return nil, err
}
opts := struct {
UID int
GID int
Mask int
}{}
if err := mnt.Decode(&opts); err != nil {
return nil, err
}
return llb.AddSecret(dest,
llb.SecretID(contents.ID()),
llb.SecretFileOpt(opts.UID, opts.GID, opts.Mask),
), nil
}
func (t *execTask) mountService(pctx *plancontext.Context, dest string, mnt *compiler.Value) (llb.RunOption, error) {
contents, err := pctx.Services.FromValue(mnt.Lookup("contents"))
if err != nil {
return nil, err
}
return llb.AddSSHSocket(
llb.SSHID(contents.ID()),
llb.SSHSocketTarget(dest),
), nil
}

View File

@ -0,0 +1,81 @@
package engine
// Execute a command in a container
#Exec: {
_type: "Exec"
// Container filesystem
input: #FS
// Transient filesystem mounts
// Key is an arbitrary name, for example "app source code"
// Value is mount configuration
mounts: [name=string]: #Mount
// Command to execute
// Example: ["echo", "hello, world!"]
args: [...string]
// Environment variables
env: [key=string]: string
// Working directory
workdir: string | *"/"
// User ID or name
user: string | *"root"
// If set, always execute even if the operation could be cached
always: true | *false
// Inject hostname resolution into the container
// key is hostname, value is IP
hosts: [hostname=string]: string
// Modified filesystem
output: #FS
// Command exit code
// Currently this field can only ever be zero.
// If the command fails, DAG execution is immediately terminated.
// FIXME: expand API to allow custom handling of failed commands
exit: int & 0
}
// A transient filesystem mount.
#Mount: {
dest: string
type: string
{
type: "cache"
contents: #CacheDir
} | {
type: "tmp"
contents: #TempDir
} | {
type: "service"
contents: #Service
} | {
type: "fs"
contents: #FS
source?: string
ro?: true | *false
} | {
type: "secret"
contents: #Secret
uid: int | *0
gid: int | *0
mask: int | *0o400
}
}
// A (best effort) persistent cache dir
#CacheDir: {
id: string
concurrency: *"shared" | "private" | "locked"
}
// A temporary directory for command execution
#TempDir: {
size: int64 | *0
}

View File

@ -24,3 +24,19 @@ setup() {
run "$DAGGER" --europa up ./writefile_failure_diff_contents.cue run "$DAGGER" --europa up ./writefile_failure_diff_contents.cue
assert_failure assert_failure
} }
@test "task: #Exec" {
cd "$TESTDIR"/tasks/exec
"$DAGGER" --europa up ./args.cue
"$DAGGER" --europa up ./env.cue
"$DAGGER" --europa up ./hosts.cue
"$DAGGER" --europa up ./mount_cache.cue
"$DAGGER" --europa up ./mount_fs.cue
TESTSECRET="hello world" "$DAGGER" --europa up ./mount_secret.cue
"$DAGGER" --europa up ./mount_tmp.cue
"$DAGGER" --europa up ./mount_service.cue
"$DAGGER" --europa up ./user.cue
"$DAGGER" --europa up ./workdir.cue
}

26
tests/tasks/exec/args.cue Normal file
View File

@ -0,0 +1,26 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
exec: engine.#Exec & {
input: image.output
args: ["sh", "-c", "echo -n hello world > /output.txt"]
}
verify: engine.#ReadFile & {
input: exec.output
path: "/output.txt"
} & {
// assert result
contents: "hello world"
}
}
}

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,3 @@
# generated by dagger
alpha.dagger.io
dagger.lock

24
tests/tasks/exec/env.cue Normal file
View File

@ -0,0 +1,24 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
verify: engine.#Exec & {
input: image.output
env: TEST: "hello world"
args: [
"sh", "-c",
#"""
test "$TEST" = "hello world"
"""#,
]
}
}
}

View File

@ -0,0 +1,25 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
verify: engine.#Exec & {
input: image.output
hosts: "unit.test": "1.2.3.4"
args: [
"sh", "-c",
#"""
grep -q "unit.test" /etc/hosts
grep -q "1.2.3.4" /etc/hosts
"""#,
]
}
}
}

View File

@ -0,0 +1,63 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
sharedCache: engine.#CacheDir & {
id: "mycache"
}
exec: engine.#Exec & {
input: image.output
mounts: cache: {
dest: "/cache"
contents: sharedCache
}
args: [
"sh", "-c",
#"""
echo -n hello world > /cache/output.txt
"""#,
]
}
verify: engine.#Exec & {
input: image.output
mounts: cache: {
dest: "/cache"
contents: sharedCache
}
args: [
"sh", "-c",
#"""
test -f /cache/output.txt
test "$(cat /cache/output.txt)" = "hello world"
"""#,
]
}
otherCache: engine.#CacheDir & {
id: "othercache"
}
verifyOtherCache: engine.#Exec & {
input: image.output
mounts: cache: {
dest: "/cache"
contents: otherCache
}
args: [
"sh", "-c",
#"""
test ! -f /cache/output.txt
"""#,
]
}
}
}

View File

@ -0,0 +1,71 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
exec: engine.#Exec & {
input: image.output
args: [
"sh", "-c",
#"""
echo -n hello world > /output.txt
"""#,
]
}
verify: engine.#Exec & {
input: image.output
mounts: fs: {
dest: "/target"
contents: exec.output
}
args: [
"sh", "-c",
#"""
test "$(cat /target/output.txt)" = "hello world"
touch /target/rw
"""#,
]
}
verifyRO: engine.#Exec & {
input: image.output
mounts: fs: {
dest: "/target"
contents: exec.output
ro: true
}
args: [
"sh", "-c",
#"""
test "$(cat /target/output.txt)" = "hello world"
touch /target/ro && exit 1
true
"""#,
]
}
verifySource: engine.#Exec & {
input: image.output
mounts: fs: {
dest: "/target.txt"
contents: exec.output
source: "/output.txt"
}
args: [
"sh", "-c",
#"""
test "$(cat /target.txt)" = "hello world"
"""#,
]
}
}
}

View File

@ -0,0 +1,49 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
context: secrets: testSecret: envvar: "TESTSECRET"
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
verify: engine.#Exec & {
input: image.output
mounts: secret: {
dest: "/run/secrets/test"
contents: context.secrets.testSecret.contents
}
args: [
"sh", "-c",
#"""
test "$(cat /run/secrets/test)" = "hello world"
ls -l /run/secrets/test | grep -- "-r--------"
"""#,
]
}
verifyPerm: engine.#Exec & {
input: image.output
mounts: secret: {
dest: "/run/secrets/test"
contents: context.secrets.testSecret.contents
uid: 42
gid: 24
mask: 0o666
}
args: [
"sh", "-c",
#"""
ls -l /run/secrets/test | grep -- "-rw-rw-rw-"
ls -l /run/secrets/test | grep -- "42"
ls -l /run/secrets/test | grep -- "24"
"""#,
]
}
}
}

View File

@ -0,0 +1,29 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
context: services: dockerSocket: unix: "/var/run/docker.sock"
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
imageWithDocker: engine.#Exec & {
input: image.output
args: ["apk", "add", "--no-cache", "docker-cli"]
}
verify: engine.#Exec & {
input: imageWithDocker.output
mounts: docker: {
dest: "/var/run/docker.sock"
contents: context.services.dockerSocket.service
}
args: ["docker", "info"]
}
}
}

View File

@ -0,0 +1,37 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
exec: engine.#Exec & {
input: image.output
mounts: temp: {
dest: "/temp"
contents: engine.#TempDir
}
args: [
"sh", "-c",
#"""
echo -n hello world > /temp/output.txt
"""#,
]
}
verify: engine.#Exec & {
input: exec.output
args: [
"sh", "-c",
#"""
test ! -f /temp/output.txt
"""#,
]
}
}
}

41
tests/tasks/exec/user.cue Normal file
View File

@ -0,0 +1,41 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
addUser: engine.#Exec & {
input: image.output
args: ["adduser", "-D", "test"]
}
verifyUsername: engine.#Exec & {
input: addUser.output
user: "test"
args: [
"sh", "-c",
#"""
test "$(whoami)" = "test"
"""#,
]
}
verifyUserID: engine.#Exec & {
input: addUser.output
user: "1000"
args: [
"sh", "-c",
#"""
test "$(whoami)" = "test"
"""#,
]
}
}
}

View File

@ -0,0 +1,24 @@
package main
import (
"alpha.dagger.io/europa/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
verify: engine.#Exec & {
input: image.output
workdir: "/tmp"
args: [
"sh", "-c",
#"""
test "$(pwd)" = "/tmp"
"""#,
]
}
}
}