Add new Client API

Signed-off-by: Helder Correia <174525+helderco@users.noreply.github.com>

plan/task/clientcommand.go

@@ -0,0 +1,165 @@
+package task
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"cuelang.org/go/cue"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("ClientCommand", func() Task { return &clientCommandTask{} })
+}
+
+type clientCommandTask struct {
+}
+
+func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	var opts struct {
+		Name string
+		Args []string
+	}
+
+	if err := v.Decode(&opts); err != nil {
+		return nil, err
+	}
+
+	flags, err := v.Lookup("flags").Fields()
+	if err != nil {
+		return nil, err
+	}
+
+	var flagArgs []string
+	for _, flag := range flags {
+		switch flag.Value.Kind() {
+		case cue.BoolKind:
+			if b, _ := flag.Value.Bool(); b {
+				flagArgs = append(flagArgs, flag.Label())
+			}
+		case cue.StringKind:
+			if s, _ := flag.Value.String(); s != "" {
+				flagArgs = append(flagArgs, flag.Label(), s)
+			}
+		}
+	}
+	opts.Args = append(flagArgs, opts.Args...)
+
+	envs, err := v.Lookup("env").Fields()
+	if err != nil {
+		return nil, err
+	}
+
+	env := make([]string, len(envs))
+	for _, envvar := range envs {
+		s, err := t.getString(pctx, envvar.Value)
+		if err != nil {
+			return nil, err
+		}
+		env = append(env, fmt.Sprintf("%s=%s", envvar.Label(), s))
+	}
+
+	lg := log.Ctx(ctx)
+	lg.Debug().Str("name", opts.Name).Str("args", strings.Join(opts.Args, " ")).Msg("running client command")
+
+	cmd := exec.CommandContext(ctx, opts.Name, opts.Args...) //#nosec G204
+	cmd.Env = append(os.Environ(), env...)
+
+	if i := v.Lookup("stdin"); i.Exists() {
+		val, err := t.getString(pctx, i)
+		if err != nil {
+			return nil, err
+		}
+
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			return nil, err
+		}
+
+		go func() {
+			defer stdin.Close()
+			io.WriteString(stdin, val)
+		}()
+	}
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	stdoutVal, err := t.readPipe(&stdout, pctx, v.Lookup("stdout"))
+	if err != nil {
+		return nil, err
+	}
+
+	stderrVal, err := t.readPipe(&stderr, pctx, v.Lookup("stderr"))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			// FIXME: stderr may be requested as a secret
+			lg.Err(err).Msg(string(exitErr.Stderr))
+		}
+		return nil, err
+	}
+
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"stdout": stdoutVal,
+		"stderr": stderrVal,
+	})
+}
+
+func (t clientCommandTask) getString(pctx *plancontext.Context, v *compiler.Value) (string, error) {
+	if plancontext.IsSecretValue(v) {
+		secret, err := pctx.Secrets.FromValue(v)
+		if err != nil {
+			return "", err
+		}
+		return secret.PlainText(), nil
+	}
+
+	s, err := v.String()
+	if err != nil {
+		return "", err
+	}
+
+	return s, nil
+}
+
+func (t clientCommandTask) readPipe(pipe *io.ReadCloser, pctx *plancontext.Context, v *compiler.Value) (*compiler.Value, error) {
+	slurp, err := io.ReadAll(*pipe)
+	if err != nil {
+		return nil, err
+	}
+
+	read := string(slurp)
+	val, _ := v.Default()
+	out := compiler.NewValue()
+
+	if plancontext.IsSecretValue(val) {
+		secret := pctx.Secrets.New(read)
+		return out.Fill(secret.MarshalCUE())
+	}
+
+	return out.Fill(read)
+}

plan/task/clientenv.go

@@ -0,0 +1,53 @@
+package task
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"cuelang.org/go/cue"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("client.env.*", func() Task { return &clientEnvTask{} })
+}
+
+type clientEnvTask struct {
+}
+
+func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	lg := log.Ctx(ctx)
+
+	envvar := v.ParentLabel(1)
+
+	lg.Debug().Str("envvar", envvar).Msg("loading environment variable")
+
+	env := os.Getenv(envvar)
+	if env == "" {
+		return nil, fmt.Errorf("environment variable %q not set", envvar)
+	}
+
+	// Resolve default in disjunction if a type hasn't been specified
+	val, _ := v.Default()
+	out := compiler.NewValue()
+
+	if plancontext.IsSecretValue(val) {
+		secret := pctx.Secrets.New(env)
+		return out.Fill(secret.MarshalCUE())
+	}
+
+	if val.IsConcrete() {
+		return nil, fmt.Errorf("unexpected concrete value, please use a type")
+	}
+
+	k := val.IncompleteKind()
+	if k == cue.StringKind {
+		return out.Fill(env)
+	}
+
+	return nil, fmt.Errorf("unsupported type %q", k)
+}

plan/task/clientfilesystemread.go

@@ -0,0 +1,199 @@
+package task
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"cuelang.org/go/cue"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("ClientFilesystemRead", func() Task { return &clientFilesystemReadTask{} })
+}
+
+type clientFilesystemReadTask struct {
+}
+
+func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
+	path, err := t.parsePath(v)
+	if err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("path %q does not exist", path)
+	}
+
+	if plancontext.IsFSValue(v.Lookup("contents")) {
+		pctx.LocalDirs.Add(path)
+	}
+
+	return nil
+}
+
+func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	path, err := t.parsePath(v)
+	if err != nil {
+		return nil, err
+	}
+
+	contents, err := t.readContents(ctx, pctx, s, v, path)
+	if err != nil {
+		return nil, err
+	}
+
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"contents": contents,
+	})
+}
+
+func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err error) {
+	path, err = v.Lookup("path").String()
+	if err != nil {
+		return
+	}
+
+	// Keep socket paths as is (e.g., npipe)
+	if plancontext.IsServiceValue(v.Lookup("contents")) {
+		return
+	}
+
+	path, err = filepath.Abs(path)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) {
+	lg := log.Ctx(ctx)
+	contents := v.Lookup("contents")
+
+	if plancontext.IsFSValue(contents) {
+		lg.Debug().Str("path", path).Msg("loading local directory")
+		return t.readFS(ctx, pctx, s, v, path)
+	}
+
+	if plancontext.IsServiceValue(contents) {
+		lg.Debug().Str("path", path).Msg("loading local service")
+		return t.readService(pctx, v, path)
+	}
+
+	if plancontext.IsSecretValue(contents) {
+		lg.Debug().Str("path", path).Msg("loading local secret file")
+		return t.readSecret(pctx, path)
+	}
+
+	if contents.IsConcrete() {
+		return nil, fmt.Errorf("unexpected concrete value, please use a type")
+	}
+
+	k := contents.IncompleteKind()
+	if k == cue.StringKind {
+		lg.Debug().Str("path", path).Msg("loading local file")
+		return t.readString(path)
+	}
+
+	return nil, fmt.Errorf("unsupported type %q", k)
+}
+
+func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
+	var dir struct {
+		Include []string
+		Exclude []string
+	}
+
+	if err := v.Decode(&dir); err != nil {
+		return nil, err
+	}
+
+	opts := []llb.LocalOption{
+		withCustomName(v, "Local %s", path),
+		// Without hint, multiple `llb.Local` operations on the
+		// same path get a different digest.
+		llb.SessionID(s.SessionID()),
+		llb.SharedKeyHint(path),
+	}
+
+	if len(dir.Include) > 0 {
+		opts = append(opts, llb.IncludePatterns(dir.Include))
+	}
+
+	// Excludes .dagger directory by default
+	excludePatterns := []string{"**/.dagger/"}
+	if len(dir.Exclude) > 0 {
+		excludePatterns = dir.Exclude
+	}
+	opts = append(opts, llb.ExcludePatterns(excludePatterns))
+
+	// FIXME: Remove the `Copy` and use `Local` directly.
+	//
+	// Copy'ing is a costly operation which should be unnecessary.
+	// However, using llb.Local directly breaks caching sometimes for unknown reasons.
+	st := llb.Scratch().File(
+		llb.Copy(
+			llb.Local(
+				path,
+				opts...,
+			),
+			"/",
+			"/",
+		),
+		withCustomName(v, "Local %s [copy]", path),
+	)
+
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return fs.MarshalCUE(), nil
+}
+
+func (t clientFilesystemReadTask) readService(pctx *plancontext.Context, v *compiler.Value, path string) (*compiler.Value, error) {
+	typ, err := v.Lookup("type").String()
+	if err != nil {
+		return nil, err
+	}
+
+	var unix, npipe string
+
+	switch typ {
+	case "unix":
+		unix = path
+	case "npipe":
+		npipe = path
+	default:
+		return nil, fmt.Errorf("invalid service type %q", typ)
+	}
+
+	service := pctx.Services.New(unix, npipe)
+	return service.MarshalCUE(), nil
+}
+
+func (t clientFilesystemReadTask) readSecret(pctx *plancontext.Context, path string) (*compiler.Value, error) {
+	contents, err := t.readString(path)
+	if err != nil {
+		return nil, err
+	}
+	secret := pctx.Secrets.New(contents)
+	return secret.MarshalCUE(), nil
+}
+
+func (t clientFilesystemReadTask) readString(path string) (string, error) {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return string(contents), nil
+}

plan/task/clientfilesystemwrite.go

@@ -0,0 +1,100 @@
+package task
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"cuelang.org/go/cue"
+	bk "github.com/moby/buildkit/client"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("ClientFilesystemWrite", func() Task { return &clientFilesystemWriteTask{} })
+}
+
+type clientFilesystemWriteTask struct {
+}
+
+func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	path, err := v.Lookup("path").String()
+	if err != nil {
+		return nil, err
+	}
+
+	path, err = filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := t.writeContents(ctx, pctx, s, v, path); err != nil {
+		return nil, err
+	}
+
+	return compiler.NewValue(), nil
+}
+
+func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
+	lg := log.Ctx(ctx)
+	contents := v.Lookup("contents")
+
+	if plancontext.IsFSValue(contents) {
+		lg.Debug().Str("path", path).Msg("writing files to local directory")
+		return t.writeFS(ctx, pctx, s, contents, path)
+	}
+
+	permissions := fs.FileMode(0644) // default permission
+	if vl := v.Lookup("permissions"); vl.Exists() {
+		p, err := vl.Int64()
+		if err != nil {
+			return err
+		}
+		permissions = fs.FileMode(p)
+	}
+
+	if plancontext.IsSecretValue(contents) {
+		lg.Debug().Str("path", path).Msg("writing secret to local file")
+		secret, err := pctx.Secrets.FromValue(contents)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(path, []byte(secret.PlainText()), permissions)
+	}
+
+	k := contents.Kind()
+	if k == cue.StringKind {
+		lg.Debug().Str("path", path).Msg("writing to local file")
+		text, err := contents.String()
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(path, []byte(text), permissions)
+	}
+
+	return fmt.Errorf("unsupported type %q", k)
+}
+
+func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
+	contents, err := pctx.FS.FromValue(v)
+	if err != nil {
+		return err
+	}
+
+	st, err := contents.State()
+	if err != nil {
+		return err
+	}
+
+	_, err = s.Export(ctx, st, nil, bk.ExportEntry{
+		Type:      bk.ExporterLocal,
+		OutputDir: path,
+	}, pctx.Platform.Get())
+
+	return err
+}

plan/task/clientplatform.go

@@ -0,0 +1,24 @@
+package task
+
+import (
+	"context"
+	"runtime"
+
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("ClientPlatform", func() Task { return &clientPlatformTask{} })
+}
+
+type clientPlatformTask struct {
+}
+
+func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"os":   runtime.GOOS,
+		"arch": runtime.GOARCH,
+	})
+}

plan/task/task.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 
 	"cuelang.org/go/cue"
@@ -20,6 +21,10 @@ var (
 		cue.Str("$dagger"),
 		cue.Str("task"),
 		cue.Hid("_name", pkg.DaggerPackage))
+	lookups = []LookupFunc{
+		defaultLookup,
+		pathLookup,
+	}
 )
 
 // State is the state of the task.
@@ -33,6 +38,7 @@ const (
 )
 
 type NewFunc func() Task
+type LookupFunc func(*compiler.Value) (Task, error)
 
 type Task interface {
 	Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error)
@@ -60,13 +66,26 @@ func New(typ string) Task {
 }
 
 func Lookup(v *compiler.Value) (Task, error) {
+	for _, lookup := range lookups {
+		t, err := lookup(v)
+		if err != nil {
+			return nil, err
+		}
+		if t != nil {
+			return t, nil
+		}
+	}
+	return nil, ErrNotTask
+}
+
+func defaultLookup(v *compiler.Value) (Task, error) {
 	if v.Kind() != cue.StructKind {
-		return nil, ErrNotTask
+		return nil, nil
 	}
 
 	typ := v.LookupPath(typePath)
 	if !typ.Exists() {
-		return nil, ErrNotTask
+		return nil, nil
 	}
 
 	typeString, err := typ.String()
@@ -78,5 +97,49 @@ func Lookup(v *compiler.Value) (Task, error) {
 	if t == nil {
 		return nil, fmt.Errorf("unknown type %q", typeString)
 	}
+
 	return t, nil
 }
+
+func pathLookup(v *compiler.Value) (Task, error) {
+	selectors := v.Path().Selectors()
+
+	// The `actions` field won't have any path based tasks since it's in user land
+	if len(selectors) == 0 || selectors[0].String() == "actions" {
+		return nil, nil
+	}
+
+	// Try an exact match first
+	if t := New(v.Path().String()); t != nil {
+		return t, nil
+	}
+
+	// FIXME: is there a way to avoid having to loop here?
+	var t Task
+	tasks.Range(func(key, value interface{}) bool {
+		if matchPathMask(selectors, key.(string)) {
+			fn := value.(NewFunc)
+			t = fn()
+			return false
+		}
+		return true
+	})
+	return t, nil
+}
+
+func matchPathMask(sels []cue.Selector, mask string) bool {
+	parts := strings.Split(mask, ".")
+	if len(sels) != len(parts) {
+		return false
+	}
+	for i, sel := range sels {
+		// use a '*' in a path mask part to match any selector
+		if parts[i] == "*" {
+			continue
+		}
+		if sel.String() != parts[i] {
+			return false
+		}
+	}
+	return true
+}