diff --git a/compiler/value.go b/compiler/value.go index cd8cb6c5..bc64e323 100644 --- a/compiler/value.go +++ b/compiler/value.go @@ -45,6 +45,14 @@ func (v *Value) FillFields(values map[string]interface{}) (*Value, error) { return v, nil } +// Fill updates a value, in place +func (v *Value) Fill(value interface{}) (*Value, error) { + if err := v.FillPath(cue.MakePath(), value); err != nil { + return nil, err + } + return v, nil +} + // LookupPath is a concurrency safe wrapper around cue.Value.LookupPath func (v *Value) LookupPath(p cue.Path) *Value { v.cc.rlock() @@ -93,6 +101,19 @@ func (f Field) Label() string { return l } +// ParentLabel returns the unquoted parent selector of a value +func (v *Value) ParentLabel(depth int) string { + sel := v.Path().Selectors() + if depth > len(sel) { + return "" + } + l := sel[len(sel)-depth].String() + if unquoted, err := strconv.Unquote(l); err == nil { + return unquoted + } + return l +} + // Proxy function to the underlying cue.Value // Field ordering is guaranteed to be stable. func (v *Value) Fields(opts ...cue.Option) ([]Field, error) { diff --git a/docs/core-concepts/1203-client.md b/docs/core-concepts/1203-client.md new file mode 100644 index 00000000..8386eb42 --- /dev/null +++ b/docs/core-concepts/1203-client.md @@ -0,0 +1,190 @@ +--- +slug: /1203/client +displayed_sidebar: europa +--- + +# Interacting with the client + +`dagger.#Plan` has a `client` field that allows interaction with the local machine where the `dagger` command line client is run. You can: + +- Read and write files and directories; +- Use local sockets; +- Load environment variables; +- Run commands; +- Get current platform. + +## Accessing the file system + +You may need to load a local directory as a `dagger.#FS` type in your plan: + +```cue +dagger.#Plan & { + // Path may be absolute, or relative to current working directory + client: filesystem: ".": read: { + // CUE type defines expected content + contents: dagger.#FS + exclude: ["node_modules"] + } + actions: { + ... + copy: docker.Copy & { + contents: client.filesystem.".".read.contents + } + ... + } +} +``` + +It’s also easy to write a file locally: + +```cue +dagger.#Plan & { + client: filesystem: "config.yaml": write: { + contents: yaml.Marshal(actions.pull.output.config) + } + actions: { + pull: docker.#Pull & { + source: "alpine" + } + } +} +``` + +## Using a local socket + +You can use a local socket in an action: + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + + +```cue +dagger.#Plan & { + client: filesystem: "/var/run/docker.sock": read: { + contents: dagger.#Service + } + + actions: { + image: alpine.#Build & { + packages: "docker-cli": {} + } + run: docker.#Run & { + input: image.output + mounts: docker: { + dest: "/var/run/docker.sock" + contents: client.filesystem."/var/run/docker.sock".read.contents + } + command: { + name: "docker" + args: ["info"] + } + } + } +} +``` + + + + + +```cue +dagger.#Plan & { + client: filesystem: "//./pipe/docker_engine": read: { + contents: dagger.#Service + type: "npipe" + } + + actions: { + image: alpine.#Build & { + packages: "docker-cli": {} + } + run: docker.#Run & { + input: image.output + mounts: docker: { + dest: "/var/run/docker.sock" + contents: client.filesystem."//./pipe/docker_engine".read.contents + } + command: { + name: "docker" + args: ["info"] + } + } + } +} +``` + + + + +## Environment variables + +Environment variables can be read from the local machine as strings or secrets, just specify the type: + +```cue +dagger.#Plan & { + client: env: { + GITLAB_USER: string + GITLAB_TOKEN: dagger.#Secret + } + actions: { + pull: docker.#Pull & { + source: "registry.gitlab.com/myuser/myrepo" + auth: { + username: client.env.GITLAB_USR + secret: client.env.GITLAB_TOKEN + } + } + } +} +``` + +## Running commands + +Sometimes you need something more advanced that only a local command can give you: + +```cue +dagger.#Plan & { + client: commands: { + os: { + name: "uname" + args: ["-s"] + } + arch: { + name: "uname" + args: ["-m"] + } + } + actions: { + build: docker.#Run & { + env: { + CLIENT_OS: client.commands.os.stdout + CLIENT_ARCH: client.commands.arch.stdout + } + } + } +} +``` + +You can also capture `stderr` for errors and provide `stdin` for input. + +## Platform + +If you need the current platform though, there’s a more portable way than running `uname` like in the previous example: + +```cue +dagger.#Plan & { + client: platform: _ + + actions: { + build: docker.#Run & { + env: { + CLIENT_OS: client.platform.os + CLIENT_ARCH: client.platform.arch + } + } + } +} +``` diff --git a/docs/core-concepts/1203-inputs.md b/docs/core-concepts/1203-inputs.md deleted file mode 100644 index c0b85edf..00000000 --- a/docs/core-concepts/1203-inputs.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -slug: /1203/inputs -displayed_sidebar: europa ---- - -# Configuring inputs diff --git a/docs/core-concepts/1204-secrets.md b/docs/core-concepts/1204-secrets.md index 9842cdfc..153270b7 100644 --- a/docs/core-concepts/1204-secrets.md +++ b/docs/core-concepts/1204-secrets.md @@ -4,3 +4,85 @@ displayed_sidebar: europa --- # How to use secrets + +Most operations in `client` support handling secrets (see [Interacting with the client](./1203-client.md)). More specifically, you can: + +- Write a secret to a file; +- Read a secret from a file; +- Read a secret from an environment variable; +- Read a secret from the output of a command; +- Use a secret as the input of a command. + +## Environmnet + +The simplest use case is reading from an environment variable: + +```cue +dagger.#Plan & { + client: env: GITHUB_TOKEN: dagger.#Secret +} +``` + +## File + +You may need to trim the whitespace, especially when reading from a file: + +```cue +dagger.#Plan & { + // Path may be absolute, or relative to current working directory + client: filesystem: ".registry": read: { + // CUE type defines expected content + contents: dagger.#Secret + } + actions: { + registry: dagger.#TrimSecret & { + input: client.filesystem.".registry".read.contents + } + pull: docker.#Pull & { + source: "myprivate/image" + auth: { + username: "_token_" + secret: registry.output + } + } + } +} +``` + +## SOPS + +There’s many ways to store encrypted secrets in your git repository. If you use [SOPS](https://github.com/mozilla/sops), here's a simple example where you can access keys from an encrypted yaml file: + +```yaml title="secrets.yaml" +myToken: ENC[AES256_GCM,data:AlUz7g==,iv:lq3mHi4GDLfAssqhPcuUIHMm5eVzJ/EpM+q7RHGCROU=,tag:dzbT5dEGhMnHbiRTu4bHdg==,type:str] +sops: + ... +``` + +```cue title="main.cue" +dagger.#Plan & { + client: commands: sops: { + name: "sops" + args: ["-d", "./secrets.yaml"] + stdout: dagger.#Secret + } + + actions: { + // Makes the yaml keys easily accessible + secrets: dagger.#DecodeSecret & { + input: client.commands.sops.stdout + format: "yaml" + } + + run: docker.#Run & { + mounts: secret: { + dest: "/run/secrets/token" + contents: secrets.output.myToken + } + // Do something with `/run/secrets/token` + ... + } + } +} + +``` diff --git a/pkg/dagger.io/dagger/plan.cue b/pkg/dagger.io/dagger/plan.cue index 72e0a43d..a6cf1c7b 100644 --- a/pkg/dagger.io/dagger/plan.cue +++ b/pkg/dagger.io/dagger/plan.cue @@ -2,6 +2,37 @@ package dagger // A special kind of program which `dagger` can execute. #Plan: { + // Access client machine + client: { + // Access client filesystem + // Path may be absolute, or relative to client working directory + filesystem: [path=string]: { + // Read data from that path + read?: _#clientFilesystemRead & { + "path": path + } + + // If set, Write to that path + write?: _#clientFilesystemWrite & { + "path": path + + // avoid race condition + if read != _|_ { + _after: read + } + } + } + + // Access client environment variables + env: [string]: *string | #Secret + + // Execute commands in the client + commands: [id=string]: _#clientCommand + + // Platform of the client machine + platform: _#clientPlatform + } + // Receive inputs from the client inputs: { // Receive directories @@ -32,6 +63,84 @@ package dagger } } +_#clientFilesystemRead: { + $dagger: task: _name: "ClientFilesystemRead" + + // Path may be absolute, or relative to client working directory + path: string + + { + // CUE type defines expected content: + // string: contents of a regular file + // #Secret: secure reference to the file contents + contents: string | #Secret + } | { + // CUE type defines expected content: + // #FS: contents of a directory + contents: #FS + + // Filename patterns to include + // Example: ["*.go", "Dockerfile"] + include?: [...string] + + // Filename patterns to exclude + // Example: ["node_modules"] + exclude?: [...string] + } | { + // CUE type defines expected content: + // #Service: unix socket or npipe + contents: #Service + + // Type of service + type: *"unix" | "npipe" + } +} + +_#clientFilesystemWrite: { + $dagger: task: _name: "ClientFilesystemWrite" + + // Path may be absolute, or relative to client working directory + path: string + { + // File contents to export (as a string or secret) + contents: string | #Secret + + // File permissions (defaults to 0o644) + permissions?: int + } | { + // Filesystem contents to export + // Reference an #FS field produced by an action + contents: #FS + } +} + +_#clientCommand: { + $dagger: task: _name: "ClientCommand" + + name: string + args: [...string] + flags: [string]: bool | string + env: [string]: string | #Secret + + // Capture standard output (as a string or secret) + stdout?: *string | #Secret + + // Capture standard error (as a string or secret) + stderr?: *string | #Secret + + // Inject standard input (from a string or secret) + stdin?: string | #Secret +} + +_#clientPlatform: { + $dagger: task: _name: "ClientPlatform" + + // Operating system of the client machine + os: string + // Hardware architecture of the client machine + arch: string +} + _#inputDirectory: { // FIXME: rename to "InputDirectory" for consistency $dagger: task: _name: "InputDirectory" diff --git a/plan/task/clientcommand.go b/plan/task/clientcommand.go new file mode 100644 index 00000000..50858c67 --- /dev/null +++ b/plan/task/clientcommand.go @@ -0,0 +1,165 @@ +package task + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "cuelang.org/go/cue" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("ClientCommand", func() Task { return &clientCommandTask{} }) +} + +type clientCommandTask struct { +} + +func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + var opts struct { + Name string + Args []string + } + + if err := v.Decode(&opts); err != nil { + return nil, err + } + + flags, err := v.Lookup("flags").Fields() + if err != nil { + return nil, err + } + + var flagArgs []string + for _, flag := range flags { + switch flag.Value.Kind() { + case cue.BoolKind: + if b, _ := flag.Value.Bool(); b { + flagArgs = append(flagArgs, flag.Label()) + } + case cue.StringKind: + if s, _ := flag.Value.String(); s != "" { + flagArgs = append(flagArgs, flag.Label(), s) + } + } + } + opts.Args = append(flagArgs, opts.Args...) + + envs, err := v.Lookup("env").Fields() + if err != nil { + return nil, err + } + + env := make([]string, len(envs)) + for _, envvar := range envs { + s, err := t.getString(pctx, envvar.Value) + if err != nil { + return nil, err + } + env = append(env, fmt.Sprintf("%s=%s", envvar.Label(), s)) + } + + lg := log.Ctx(ctx) + lg.Debug().Str("name", opts.Name).Str("args", strings.Join(opts.Args, " ")).Msg("running client command") + + cmd := exec.CommandContext(ctx, opts.Name, opts.Args...) //#nosec G204 + cmd.Env = append(os.Environ(), env...) + + if i := v.Lookup("stdin"); i.Exists() { + val, err := t.getString(pctx, i) + if err != nil { + return nil, err + } + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + + go func() { + defer stdin.Close() + io.WriteString(stdin, val) + }() + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + if err := cmd.Start(); err != nil { + return nil, err + } + + stdoutVal, err := t.readPipe(&stdout, pctx, v.Lookup("stdout")) + if err != nil { + return nil, err + } + + stderrVal, err := t.readPipe(&stderr, pctx, v.Lookup("stderr")) + if err != nil { + return nil, err + } + + if err := cmd.Wait(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + // FIXME: stderr may be requested as a secret + lg.Err(err).Msg(string(exitErr.Stderr)) + } + return nil, err + } + + return compiler.NewValue().FillFields(map[string]interface{}{ + "stdout": stdoutVal, + "stderr": stderrVal, + }) +} + +func (t clientCommandTask) getString(pctx *plancontext.Context, v *compiler.Value) (string, error) { + if plancontext.IsSecretValue(v) { + secret, err := pctx.Secrets.FromValue(v) + if err != nil { + return "", err + } + return secret.PlainText(), nil + } + + s, err := v.String() + if err != nil { + return "", err + } + + return s, nil +} + +func (t clientCommandTask) readPipe(pipe *io.ReadCloser, pctx *plancontext.Context, v *compiler.Value) (*compiler.Value, error) { + slurp, err := io.ReadAll(*pipe) + if err != nil { + return nil, err + } + + read := string(slurp) + val, _ := v.Default() + out := compiler.NewValue() + + if plancontext.IsSecretValue(val) { + secret := pctx.Secrets.New(read) + return out.Fill(secret.MarshalCUE()) + } + + return out.Fill(read) +} diff --git a/plan/task/clientenv.go b/plan/task/clientenv.go new file mode 100644 index 00000000..82a13f81 --- /dev/null +++ b/plan/task/clientenv.go @@ -0,0 +1,53 @@ +package task + +import ( + "context" + "fmt" + "os" + + "cuelang.org/go/cue" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("client.env.*", func() Task { return &clientEnvTask{} }) +} + +type clientEnvTask struct { +} + +func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + lg := log.Ctx(ctx) + + envvar := v.ParentLabel(1) + + lg.Debug().Str("envvar", envvar).Msg("loading environment variable") + + env := os.Getenv(envvar) + if env == "" { + return nil, fmt.Errorf("environment variable %q not set", envvar) + } + + // Resolve default in disjunction if a type hasn't been specified + val, _ := v.Default() + out := compiler.NewValue() + + if plancontext.IsSecretValue(val) { + secret := pctx.Secrets.New(env) + return out.Fill(secret.MarshalCUE()) + } + + if val.IsConcrete() { + return nil, fmt.Errorf("unexpected concrete value, please use a type") + } + + k := val.IncompleteKind() + if k == cue.StringKind { + return out.Fill(env) + } + + return nil, fmt.Errorf("unsupported type %q", k) +} diff --git a/plan/task/clientfilesystemread.go b/plan/task/clientfilesystemread.go new file mode 100644 index 00000000..211b4572 --- /dev/null +++ b/plan/task/clientfilesystemread.go @@ -0,0 +1,199 @@ +package task + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "cuelang.org/go/cue" + "github.com/moby/buildkit/client/llb" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("ClientFilesystemRead", func() Task { return &clientFilesystemReadTask{} }) +} + +type clientFilesystemReadTask struct { +} + +func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error { + path, err := t.parsePath(v) + if err != nil { + return err + } + + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("path %q does not exist", path) + } + + if plancontext.IsFSValue(v.Lookup("contents")) { + pctx.LocalDirs.Add(path) + } + + return nil +} + +func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + path, err := t.parsePath(v) + if err != nil { + return nil, err + } + + contents, err := t.readContents(ctx, pctx, s, v, path) + if err != nil { + return nil, err + } + + return compiler.NewValue().FillFields(map[string]interface{}{ + "contents": contents, + }) +} + +func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err error) { + path, err = v.Lookup("path").String() + if err != nil { + return + } + + // Keep socket paths as is (e.g., npipe) + if plancontext.IsServiceValue(v.Lookup("contents")) { + return + } + + path, err = filepath.Abs(path) + if err != nil { + return + } + + return +} + +func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) { + lg := log.Ctx(ctx) + contents := v.Lookup("contents") + + if plancontext.IsFSValue(contents) { + lg.Debug().Str("path", path).Msg("loading local directory") + return t.readFS(ctx, pctx, s, v, path) + } + + if plancontext.IsServiceValue(contents) { + lg.Debug().Str("path", path).Msg("loading local service") + return t.readService(pctx, v, path) + } + + if plancontext.IsSecretValue(contents) { + lg.Debug().Str("path", path).Msg("loading local secret file") + return t.readSecret(pctx, path) + } + + if contents.IsConcrete() { + return nil, fmt.Errorf("unexpected concrete value, please use a type") + } + + k := contents.IncompleteKind() + if k == cue.StringKind { + lg.Debug().Str("path", path).Msg("loading local file") + return t.readString(path) + } + + return nil, fmt.Errorf("unsupported type %q", k) +} + +func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) { + var dir struct { + Include []string + Exclude []string + } + + if err := v.Decode(&dir); err != nil { + return nil, err + } + + opts := []llb.LocalOption{ + withCustomName(v, "Local %s", path), + // Without hint, multiple `llb.Local` operations on the + // same path get a different digest. + llb.SessionID(s.SessionID()), + llb.SharedKeyHint(path), + } + + if len(dir.Include) > 0 { + opts = append(opts, llb.IncludePatterns(dir.Include)) + } + + // Excludes .dagger directory by default + excludePatterns := []string{"**/.dagger/"} + if len(dir.Exclude) > 0 { + excludePatterns = dir.Exclude + } + opts = append(opts, llb.ExcludePatterns(excludePatterns)) + + // FIXME: Remove the `Copy` and use `Local` directly. + // + // Copy'ing is a costly operation which should be unnecessary. + // However, using llb.Local directly breaks caching sometimes for unknown reasons. + st := llb.Scratch().File( + llb.Copy( + llb.Local( + path, + opts..., + ), + "/", + "/", + ), + withCustomName(v, "Local %s [copy]", path), + ) + + result, err := s.Solve(ctx, st, pctx.Platform.Get()) + if err != nil { + return nil, err + } + + fs := pctx.FS.New(result) + return fs.MarshalCUE(), nil +} + +func (t clientFilesystemReadTask) readService(pctx *plancontext.Context, v *compiler.Value, path string) (*compiler.Value, error) { + typ, err := v.Lookup("type").String() + if err != nil { + return nil, err + } + + var unix, npipe string + + switch typ { + case "unix": + unix = path + case "npipe": + npipe = path + default: + return nil, fmt.Errorf("invalid service type %q", typ) + } + + service := pctx.Services.New(unix, npipe) + return service.MarshalCUE(), nil +} + +func (t clientFilesystemReadTask) readSecret(pctx *plancontext.Context, path string) (*compiler.Value, error) { + contents, err := t.readString(path) + if err != nil { + return nil, err + } + secret := pctx.Secrets.New(contents) + return secret.MarshalCUE(), nil +} + +func (t clientFilesystemReadTask) readString(path string) (string, error) { + contents, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(contents), nil +} diff --git a/plan/task/clientfilesystemwrite.go b/plan/task/clientfilesystemwrite.go new file mode 100644 index 00000000..67f4c967 --- /dev/null +++ b/plan/task/clientfilesystemwrite.go @@ -0,0 +1,100 @@ +package task + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + + "cuelang.org/go/cue" + bk "github.com/moby/buildkit/client" + "github.com/rs/zerolog/log" + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("ClientFilesystemWrite", func() Task { return &clientFilesystemWriteTask{} }) +} + +type clientFilesystemWriteTask struct { +} + +func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) { + path, err := v.Lookup("path").String() + if err != nil { + return nil, err + } + + path, err = filepath.Abs(path) + if err != nil { + return nil, err + } + + if err := t.writeContents(ctx, pctx, s, v, path); err != nil { + return nil, err + } + + return compiler.NewValue(), nil +} + +func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { + lg := log.Ctx(ctx) + contents := v.Lookup("contents") + + if plancontext.IsFSValue(contents) { + lg.Debug().Str("path", path).Msg("writing files to local directory") + return t.writeFS(ctx, pctx, s, contents, path) + } + + permissions := fs.FileMode(0644) // default permission + if vl := v.Lookup("permissions"); vl.Exists() { + p, err := vl.Int64() + if err != nil { + return err + } + permissions = fs.FileMode(p) + } + + if plancontext.IsSecretValue(contents) { + lg.Debug().Str("path", path).Msg("writing secret to local file") + secret, err := pctx.Secrets.FromValue(contents) + if err != nil { + return err + } + return os.WriteFile(path, []byte(secret.PlainText()), permissions) + } + + k := contents.Kind() + if k == cue.StringKind { + lg.Debug().Str("path", path).Msg("writing to local file") + text, err := contents.String() + if err != nil { + return err + } + return os.WriteFile(path, []byte(text), permissions) + } + + return fmt.Errorf("unsupported type %q", k) +} + +func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error { + contents, err := pctx.FS.FromValue(v) + if err != nil { + return err + } + + st, err := contents.State() + if err != nil { + return err + } + + _, err = s.Export(ctx, st, nil, bk.ExportEntry{ + Type: bk.ExporterLocal, + OutputDir: path, + }, pctx.Platform.Get()) + + return err +} diff --git a/plan/task/clientplatform.go b/plan/task/clientplatform.go new file mode 100644 index 00000000..4fc2ad06 --- /dev/null +++ b/plan/task/clientplatform.go @@ -0,0 +1,24 @@ +package task + +import ( + "context" + "runtime" + + "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" + "go.dagger.io/dagger/solver" +) + +func init() { + Register("ClientPlatform", func() Task { return &clientPlatformTask{} }) +} + +type clientPlatformTask struct { +} + +func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) { + return compiler.NewValue().FillFields(map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + }) +} diff --git a/plan/task/task.go b/plan/task/task.go index 408253a7..e7981663 100644 --- a/plan/task/task.go +++ b/plan/task/task.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "sync" "cuelang.org/go/cue" @@ -20,6 +21,10 @@ var ( cue.Str("$dagger"), cue.Str("task"), cue.Hid("_name", pkg.DaggerPackage)) + lookups = []LookupFunc{ + defaultLookup, + pathLookup, + } ) // State is the state of the task. @@ -33,6 +38,7 @@ const ( ) type NewFunc func() Task +type LookupFunc func(*compiler.Value) (Task, error) type Task interface { Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) @@ -60,13 +66,26 @@ func New(typ string) Task { } func Lookup(v *compiler.Value) (Task, error) { + for _, lookup := range lookups { + t, err := lookup(v) + if err != nil { + return nil, err + } + if t != nil { + return t, nil + } + } + return nil, ErrNotTask +} + +func defaultLookup(v *compiler.Value) (Task, error) { if v.Kind() != cue.StructKind { - return nil, ErrNotTask + return nil, nil } typ := v.LookupPath(typePath) if !typ.Exists() { - return nil, ErrNotTask + return nil, nil } typeString, err := typ.String() @@ -78,5 +97,49 @@ func Lookup(v *compiler.Value) (Task, error) { if t == nil { return nil, fmt.Errorf("unknown type %q", typeString) } + return t, nil } + +func pathLookup(v *compiler.Value) (Task, error) { + selectors := v.Path().Selectors() + + // The `actions` field won't have any path based tasks since it's in user land + if len(selectors) == 0 || selectors[0].String() == "actions" { + return nil, nil + } + + // Try an exact match first + if t := New(v.Path().String()); t != nil { + return t, nil + } + + // FIXME: is there a way to avoid having to loop here? + var t Task + tasks.Range(func(key, value interface{}) bool { + if matchPathMask(selectors, key.(string)) { + fn := value.(NewFunc) + t = fn() + return false + } + return true + }) + return t, nil +} + +func matchPathMask(sels []cue.Selector, mask string) bool { + parts := strings.Split(mask, ".") + if len(sels) != len(parts) { + return false + } + for i, sel := range sels { + // use a '*' in a path mask part to match any selector + if parts[i] == "*" { + continue + } + if sel.String() != parts[i] { + return false + } + } + return true +} diff --git a/tests/plan.bats b/tests/plan.bats index d356f8c1..a00f5eec 100644 --- a/tests/plan.bats +++ b/tests/plan.bats @@ -10,6 +10,142 @@ setup() { "$DAGGER" "do" -p ./plan/hello-europa test } +@test "plan/client/filesystem/read/fs/usage" { + cd "$TESTDIR/plan/client/filesystem/read/fs" + + "$DAGGER" "do" -p ./usage test valid + + run "$DAGGER" "do" -p ./usage test conflictingValues + assert_failure + assert_output --partial 'conflicting values "local directory" and "local foobar"' + + run "$DAGGER" "do" -p ./usage test excluded + assert_failure + assert_line --partial 'test.log: no such file or directory' + + run "$DAGGER" "do" -p ./usage test notExists + assert_failure + assert_output --partial 'test.json: no such file or directory' +} + +@test "plan/client/filesystem/read/fs/not_exists" { + cd "$TESTDIR/plan/client/filesystem/read/fs/not_exists" + + run "$DAGGER" "do" -p . test + assert_failure + assert_output --partial 'path "/foobar" does not exist' +} + +@test "plan/client/filesystem/read/fs/relative" { + cd "$TESTDIR/plan/client/filesystem/read/fs/relative" + + "$DAGGER" "do" -p . test valid + + run "$DAGGER" "do" -p . test notIncluded + assert_failure + assert_output --partial 'test.log: no such file or directory' +} + +@test "plan/client/filesystem/read/file" { + cd "$TESTDIR/plan/client/filesystem/read/file" + + "$DAGGER" "do" -p . test usage + + run "$DAGGER" "do" -p . test concrete + assert_failure + assert_output --partial "unexpected concrete value" +} + +@test "plan/client/filesystem/read/service" { + cd "$TESTDIR" + "$DAGGER" "do" -p ./plan/client/filesystem/read/service/valid.cue test + + run "$DAGGER" "do" -p ./plan/client/filesystem/read/service/invalid.cue test + assert_failure +} + +@test "plan/client/filesystem/write fs" { + cd "$TESTDIR/plan/client/filesystem/write" + + rm -rf "./out_fs" + + "$DAGGER" "do" -p . test fs + assert [ "$(cat ./out_fs/test)" = "foobar" ] + + rm -rf "./out_fs" +} + +@test "plan/client/filesystem/write files" { + cd "$TESTDIR/plan/client/filesystem/write" + + mkdir -p ./out_files + rm -f ./out_files/* + + # -- string -- + + "$DAGGER" "do" -p ./ test file + + assert [ "$(cat ./out_files/test.txt)" = "foobaz" ] + run ls -l "./out_files/test.txt" + assert_output --partial "-rw-r--r--" + + # -- secret -- + + "$DAGGER" "do" -p ./ test secret + + assert [ "$(cat ./out_files/secret.txt)" = "foo-barab-oof" ] + run ls -l "./out_files/secret.txt" + assert_output --partial "-rw-------" + + rm -rf ./out_files +} + +@test "plan/client/filesystem/conflict" { + cd "$TESTDIR/plan/client/filesystem/conflict" + + echo -n foo > test.txt + run "$DAGGER" "do" --log-level debug -p . test + assert_line --regexp "client\.filesystem\..+\.write.+dependency=client\.filesystem\..+\.read" + + rm -f test.txt +} + +@test "plan/client/env usage" { + cd "${TESTDIR}" + + export TEST_STRING="foo" + export TEST_SECRET="bar" + + "$DAGGER" "do" -p ./plan/client/env test usage +} + +@test "plan/client/env not exists" { + cd "${TESTDIR}" + + run "$DAGGER" "do" -p ./plan/client/env test usage + assert_failure +} + +@test "plan/client/env invalid" { + cd "${TESTDIR}" + + export TEST_FAIL="foobar" + + run "$DAGGER" "do" -p ./plan/client/env test concrete + assert_failure + assert_output --partial "TEST_FAIL: unexpected concrete value" +} + +@test "plan/client/commands" { + cd "${TESTDIR}/plan/client/commands" + + "$DAGGER" "do" -p . test valid + + run "$DAGGER" "do" -p . test invalid + assert_failure + assert_output --partial 'exec: "foobar": executable file not found' +} + @test "plan/proxy invalid schema" { cd "$TESTDIR" run "$DAGGER" "do" -p ./plan/proxy/invalid_schema.cue verify diff --git a/tests/plan/client/commands/test.cue b/tests/plan/client/commands/test.cue new file mode 100644 index 00000000..d93b30e3 --- /dev/null +++ b/tests/plan/client/commands/test.cue @@ -0,0 +1,68 @@ +package main + +import ( + "strings" + "dagger.io/dagger" +) + +dagger.#Plan & { + client: commands: { + normal: { + name: "echo" + args: ["hello europa"] + } + relative: { + name: "cat" + args: ["./test.txt"] + } + secret: { + name: "tee" + stdout: dagger.#Secret + stdin: "hello secretive europa" + } + error: { + name: "sh" + flags: "-c": ">&2 echo 'error'" + stderr: string + } + invalid: name: "foobar" + } + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + test: { + invalid: dagger.#Exec & { + input: image.output + args: ["echo", client.commands.invalid.stdout] + } + valid: { + normal: dagger.#Exec & { + input: image.output + args: ["test", strings.TrimSpace(client.commands.normal.stdout), "=", "hello europa"] + } + relative: dagger.#Exec & { + input: image.output + args: ["test", strings.TrimSpace(client.commands.relative.stdout), "=", "test"] + } + error: dagger.#Exec & { + input: image.output + args: ["test", strings.TrimSpace(client.commands.error.stderr), "=", "error"] + } + secret: dagger.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: client.commands.secret.stdout + } + args: [ + "sh", "-c", + #""" + test "$(cat /run/secrets/test)" = "hello secretive europa" + """#, + ] + } + } + } + } +} diff --git a/tests/plan/client/commands/test.txt b/tests/plan/client/commands/test.txt new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/tests/plan/client/commands/test.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/tests/plan/client/env/test.cue b/tests/plan/client/env/test.cue new file mode 100644 index 00000000..af24b219 --- /dev/null +++ b/tests/plan/client/env/test.cue @@ -0,0 +1,44 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: env: { + TEST_STRING: string + TEST_SECRET: dagger.#Secret + TEST_FAIL: "env" + } + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + test: { + concrete: dagger.#Exec & { + input: image.output + args: [client.env.TEST_FAIL] + } + usage: { + string: dagger.#Exec & { + input: image.output + args: ["test", client.env.TEST_STRING, "=", "foo"] + } + secret: dagger.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: client.env.TEST_SECRET + } + args: [ + "sh", "-c", + #""" + test "$(cat /run/secrets/test)" = "bar" + ls -l /run/secrets/test | grep -- "-r--------" + """#, + ] + } + } + } + } +} diff --git a/tests/plan/client/filesystem/conflict/test.cue b/tests/plan/client/filesystem/conflict/test.cue new file mode 100644 index 00000000..a6540e03 --- /dev/null +++ b/tests/plan/client/filesystem/conflict/test.cue @@ -0,0 +1,41 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "test.txt": { + // no dependencies between these two, one must be forced + read: contents: string + write: contents: actions.test.export.contents + } + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + test: { + read: dagger.#Exec & { + input: image.output + args: ["echo", client.filesystem."test.txt".read.contents] + } + write: dagger.#Exec & { + input: image.output + args: ["sh", "-c", + #""" + echo -n bar > /out.txt + """#, + ] + } + export: dagger.#ReadFile & { + input: write.output + path: "out.txt" + } + // FIXME: hack until we can do outputs with `dagger do` + verify: dagger.#Exec & { + input: image.output + args: ["echo", client.filesystem."test.txt".write.contents] + } + } + } +} diff --git a/tests/plan/client/filesystem/read/file/cmd.sh b/tests/plan/client/filesystem/read/file/cmd.sh new file mode 100644 index 00000000..81c93fbf --- /dev/null +++ b/tests/plan/client/filesystem/read/file/cmd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +env diff --git a/tests/plan/client/filesystem/read/file/secret.txt b/tests/plan/client/filesystem/read/file/secret.txt new file mode 100644 index 00000000..ba0e162e --- /dev/null +++ b/tests/plan/client/filesystem/read/file/secret.txt @@ -0,0 +1 @@ +bar \ No newline at end of file diff --git a/tests/plan/client/filesystem/read/file/test.cue b/tests/plan/client/filesystem/read/file/test.cue new file mode 100644 index 00000000..6d608238 --- /dev/null +++ b/tests/plan/client/filesystem/read/file/test.cue @@ -0,0 +1,44 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: { + "cmd.sh": read: contents: "env" + "test.txt": read: contents: string + "secret.txt": read: contents: dagger.#Secret + } + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + test: { + concrete: dagger.#Exec & { + input: image.output + args: ["sh", "-c", client.filesystem."cmd.sh".read.contents] + } + usage: { + string: dagger.#Exec & { + input: image.output + args: ["test", client.filesystem."test.txt".read.contents, "=", "foo"] + } + secret: dagger.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: client.filesystem."secret.txt".read.contents + } + args: [ + "sh", "-c", + #""" + test "$(cat /run/secrets/test)" = "bar" + ls -l /run/secrets/test | grep -- "-r--------" + """#, + ] + } + } + } + } +} diff --git a/tests/plan/client/filesystem/read/file/test.txt b/tests/plan/client/filesystem/read/file/test.txt new file mode 100644 index 00000000..19102815 --- /dev/null +++ b/tests/plan/client/filesystem/read/file/test.txt @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/tests/plan/client/filesystem/read/fs/not_exists/test.cue b/tests/plan/client/filesystem/read/fs/not_exists/test.cue new file mode 100644 index 00000000..10c7d6e0 --- /dev/null +++ b/tests/plan/client/filesystem/read/fs/not_exists/test.cue @@ -0,0 +1,11 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "/foobar": read: contents: dagger.#FS + actions: test: { + } +} diff --git a/tests/plan/client/filesystem/read/fs/relative/test.cue b/tests/plan/client/filesystem/read/fs/relative/test.cue new file mode 100644 index 00000000..5dc984ad --- /dev/null +++ b/tests/plan/client/filesystem/read/fs/relative/test.cue @@ -0,0 +1,22 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "../rootfs": read: { + contents: dagger.#FS + include: ["*.txt"] + } + actions: test: { + [string]: dagger.#ReadFile & { + input: client.filesystem."../rootfs".read.contents + } + valid: { + path: "test.txt" + contents: "local directory" + } + notIncluded: path: "test.log" + } +} diff --git a/tests/plan/client/filesystem/read/fs/rootfs/test.log b/tests/plan/client/filesystem/read/fs/rootfs/test.log new file mode 100644 index 00000000..bbde3dc9 --- /dev/null +++ b/tests/plan/client/filesystem/read/fs/rootfs/test.log @@ -0,0 +1 @@ +excluded diff --git a/tests/plan/client/filesystem/read/fs/rootfs/test.txt b/tests/plan/client/filesystem/read/fs/rootfs/test.txt new file mode 100644 index 00000000..85b0daa3 --- /dev/null +++ b/tests/plan/client/filesystem/read/fs/rootfs/test.txt @@ -0,0 +1 @@ +local directory \ No newline at end of file diff --git a/tests/plan/client/filesystem/read/fs/usage/test.cue b/tests/plan/client/filesystem/read/fs/usage/test.cue new file mode 100644 index 00000000..792d69a5 --- /dev/null +++ b/tests/plan/client/filesystem/read/fs/usage/test.cue @@ -0,0 +1,27 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: rootfs: read: { + contents: dagger.#FS + exclude: ["*.log"] + } + actions: test: { + [string]: dagger.#ReadFile & { + input: client.filesystem.rootfs.read.contents + } + valid: { + path: "test.txt" + contents: "local directory" + } + conflictingValues: { + path: "test.txt" + contents: "local foobar" + } + excluded: path: "test.log" + notExists: path: "test.json" + } +} diff --git a/tests/plan/client/filesystem/read/service/invalid.cue b/tests/plan/client/filesystem/read/service/invalid.cue new file mode 100644 index 00000000..90125fcb --- /dev/null +++ b/tests/plan/client/filesystem/read/service/invalid.cue @@ -0,0 +1,29 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "/var/run/docker.soc": read: contents: dagger.#Service + + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + imageWithDocker: dagger.#Exec & { + input: image.output + args: ["apk", "add", "--no-cache", "docker-cli"] + } + + test: dagger.#Exec & { + input: imageWithDocker.output + mounts: docker: { + dest: "/var/run/docker.sock" + contents: client.filesystem."/var/run/docker.soc".read.contents + } + args: ["docker", "info"] + } + } +} diff --git a/tests/plan/client/filesystem/read/service/valid.cue b/tests/plan/client/filesystem/read/service/valid.cue new file mode 100644 index 00000000..3954244b --- /dev/null +++ b/tests/plan/client/filesystem/read/service/valid.cue @@ -0,0 +1,29 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "/var/run/docker.sock": read: contents: dagger.#Service + + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + imageWithDocker: dagger.#Exec & { + input: image.output + args: ["apk", "add", "--no-cache", "docker-cli"] + } + + test: dagger.#Exec & { + input: imageWithDocker.output + mounts: docker: { + dest: "/var/run/docker.sock" + contents: client.filesystem."/var/run/docker.sock".read.contents + } + args: ["docker", "info"] + } + } +} diff --git a/tests/plan/client/filesystem/read/service/windows.cue b/tests/plan/client/filesystem/read/service/windows.cue new file mode 100644 index 00000000..2c383387 --- /dev/null +++ b/tests/plan/client/filesystem/read/service/windows.cue @@ -0,0 +1,29 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: "//./pipe/docker_engine": read: contents: dagger.#Service + + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + + imageWithDocker: dagger.#Exec & { + input: image.output + args: ["apk", "add", "--no-cache", "docker-cli"] + } + + test: dagger.#Exec & { + input: imageWithDocker.output + mounts: docker: { + dest: "/var/run/docker.sock" + contents: client.filesystem."//./pipe/docker_engine".read.contents + } + args: ["docker", "info"] + } + } +} diff --git a/tests/plan/client/filesystem/write/test.cue b/tests/plan/client/filesystem/write/test.cue new file mode 100644 index 00000000..2be7c237 --- /dev/null +++ b/tests/plan/client/filesystem/write/test.cue @@ -0,0 +1,69 @@ +package main + +import ( + "dagger.io/dagger" +) + +dagger.#Plan & { + client: filesystem: { + out_fs: write: contents: actions.test.fs.data.output + "out_files/test.txt": write: contents: actions.test.file.data.contents + "out_files/secret.txt": write: { + contents: actions.test.secret.data.output + permissions: 0o600 + } + } + + actions: { + image: dagger.#Pull & { + source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3" + } + test: { + fs: { + data: dagger.#WriteFile & { + input: dagger.#Scratch + path: "/test" + contents: "foobar" + } + // FIXME: hack until we can do outputs with `dagger do` + verify: dagger.#ReadFile & { + input: client.filesystem."out_fs".write.contents + path: "test" + } + } + file: { + // Only using contents for reference in client + data: dagger.#WriteFile & { + input: dagger.#Scratch + path: "/test" + contents: "foobaz" + } + // FIXME: hack until we can do outputs with `dagger do` + verify: dagger.#Exec & { + input: image.output + args: ["echo", "-c", client.filesystem."out_files/test.txt".write.contents] + } + } + secret: { + create: dagger.#WriteFile & { + input: dagger.#Scratch + path: "/test" + contents: "foo-barab-oof" + } + data: dagger.#NewSecret & { + input: create.output + path: "/test" + } + // FIXME: hack until we can do outputs with `dagger do` + verify: dagger.#Exec & { + input: image.output + mounts: secret: { + dest: "/run/secrets/test" + contents: client.filesystem."out_files/secret.txt".write.contents + } + args: ["id"] + } + } + } + } +} diff --git a/website/sidebars.js b/website/sidebars.js index 0b3fe829..5e1bb95a 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -88,7 +88,7 @@ module.exports = { collapsed: false, items: [ "core-concepts/plan", - "core-concepts/inputs", + "core-concepts/client", "core-concepts/secrets", "core-concepts/caching", "core-concepts/container-images",