From 5b7b1cab79148dd7da9399667fdcae352e86ac1f Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 30 Nov 2021 17:51:28 -0800 Subject: [PATCH 1/7] Migrate dagger.#Secret and dagger.#Stream to new format - Refactored to keep every transformation of built-in types (e.g. FS, Secret, etc) to/from CUE in the same place (plancontext) - dagger.#Service and dagger.#Secret are now following the new FS-like format (e.g. `_service: id: string`) - Backward compatibility - dagger.#Stream is now an alias for dagger.#Service Signed-off-by: Andrea Luzzardi --- client/client.go | 2 +- cmd/dagger/cmd/common/common.go | 6 +- compiler/compiler.go | 12 ---- docs/reference/argocd.md | 16 ++--- docs/reference/dagger/README.md | 12 ++++ docs/reference/git.md | 6 +- docs/reference/kubernetes/README.md | 2 +- docs/reference/kubernetes/helm.md | 2 +- environment/pipeline.go | 56 +++++------------ plan/plan.go | 5 +- plan/task/import.go | 15 ++--- plan/task/secretenv.go | 14 ++--- plan/task/secretfile.go | 13 ++-- plancontext/context.go | 29 ++++----- plancontext/context_test.go | 10 ++- plancontext/fs.go | 55 ++++++++++++++--- plancontext/localdir.go | 35 ++--------- plancontext/secret.go | 71 +++++++++++++++++++--- plancontext/service.go | 81 +++++++++++++++++++++---- solver/secretsprovider.go | 4 +- solver/socketprovider.go | 2 +- solver/socketprovider_unix.go | 4 +- solver/socketprovider_windows.go | 4 +- state/input.go | 20 ++---- stdlib/dagger/dagger.cue | 19 +++--- tests/compute/secrets/simple/simple.cue | 2 - tests/core.bats | 4 -- 27 files changed, 284 insertions(+), 217 deletions(-) diff --git a/client/client.go b/client/client.go index 89fb6139..4e411743 100644 --- a/client/client.go +++ b/client/client.go @@ -217,7 +217,7 @@ func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, secureSprintf := func(format string, a ...interface{}) string { s := fmt.Sprintf(format, a...) for _, secret := range secrets { - s = strings.ReplaceAll(s, secret.PlainText, "***") + s = strings.ReplaceAll(s, secret.PlainText(), "***") } return s } diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index c9955894..bb875234 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -88,7 +88,11 @@ func FormatValue(val *compiler.Value) string { if val.HasAttr("artifact") { return "dagger.#Artifact" } - if val.HasAttr("secret") { + + if val.LookupPath(cue.MakePath( + cue.Hid("_secret", "alpha.dagger.io/dagger"), + cue.Str("id"), + )).Exists() { return "dagger.#Secret" } if val.IsConcreteR() != nil { diff --git a/compiler/compiler.go b/compiler/compiler.go index 1dd4b7c8..0f3c5784 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -24,10 +24,6 @@ func NewValue() *Value { return DefaultCompiler.NewValue() } -func NewValueWithContent(x interface{}, selectors ...cue.Selector) (*Value, error) { - return DefaultCompiler.NewValueWithContent(x, selectors...) -} - // FIXME can be refactored away now? func Wrap(v cue.Value) *Value { return DefaultCompiler.Wrap(v) @@ -84,14 +80,6 @@ func (c *Compiler) NewValue() *Value { return empty } -func (c *Compiler) NewValueWithContent(x interface{}, selectors ...cue.Selector) (*Value, error) { - v := c.NewValue() - if err := v.FillPath(cue.MakePath(selectors...), x); err != nil { - return nil, err - } - return v, nil -} - func (c *Compiler) Compile(name string, src string) (*Value, error) { c.lock() defer c.unlock() diff --git a/docs/reference/argocd.md b/docs/reference/argocd.md index c29bb096..11264085 100644 --- a/docs/reference/argocd.md +++ b/docs/reference/argocd.md @@ -21,7 +21,7 @@ Create an ArgoCD application |*config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*config.server* | `string` |ArgoCD server | |*config.project* | `*"default" \| string` |ArgoCD project | -|*config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*config.token* | `*null \| {}` |ArgoCD authentication token | |*name* | `string` |App name | |*repo* | `string` |Repository url (git or helm) | |*path* | `string` |Folder to deploy | @@ -29,7 +29,7 @@ Create an ArgoCD application |*image.config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*image.config.server* | `string` |ArgoCD server | |*image.config.project* | `*"default" \| string` |ArgoCD project | -|*image.config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*image.config.token* | `*null \| {}` |ArgoCD authentication token | |*namespace* | `*"default" \| string` |Destination namespace | |*env.APP_NAME* | `string` |- | |*env.APP_REPO* | `string` |- | @@ -52,7 +52,7 @@ Re-usable CLI component |*config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*config.server* | `string` |ArgoCD server | |*config.project* | `*"default" \| string` |ArgoCD project | -|*config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*config.token* | `*null \| {}` |ArgoCD authentication token | ### argocd.#CLI Outputs @@ -69,7 +69,7 @@ ArgoCD configuration |*version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*server* | `string` |ArgoCD server | |*project* | `*"default" \| string` |ArgoCD project | -|*token* | `dagger.#Secret` |ArgoCD authentication token | +|*token* | `*null \| {}` |ArgoCD authentication token | ### argocd.#Config Outputs @@ -86,7 +86,7 @@ Get application's status |*config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*config.server* | `string` |ArgoCD server | |*config.project* | `*"default" \| string` |ArgoCD project | -|*config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*config.token* | `*null \| {}` |ArgoCD authentication token | |*name* | `string` |ArgoCD application | ### argocd.#Status Outputs @@ -111,18 +111,18 @@ Sync an application to its targer state |*config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*config.server* | `string` |ArgoCD server | |*config.project* | `*"default" \| string` |ArgoCD project | -|*config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*config.token* | `*null \| {}` |ArgoCD authentication token | |*application* | `string` |ArgoCD application | |*wait* | `*false \| bool` |Wait the application to sync correctly | |*ctr.image.config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*ctr.image.config.server* | `string` |ArgoCD server | |*ctr.image.config.project* | `*"default" \| string` |ArgoCD project | -|*ctr.image.config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*ctr.image.config.token* | `*null \| {}` |ArgoCD authentication token | |*ctr.env.APPLICATION* | `string` |- | |*status.config.version* | `*"v2.0.5" \| string` |ArgoCD CLI binary version | |*status.config.server* | `string` |ArgoCD server | |*status.config.project* | `*"default" \| string` |ArgoCD project | -|*status.config.token* | `dagger.#Secret` |ArgoCD authentication token | +|*status.config.token* | `*null \| {}` |ArgoCD authentication token | |*status.name* | `string` |ArgoCD application | ### argocd.#Sync Outputs diff --git a/docs/reference/dagger/README.md b/docs/reference/dagger/README.md index 3f545bc6..fd9a775f 100644 --- a/docs/reference/dagger/README.md +++ b/docs/reference/dagger/README.md @@ -56,6 +56,18 @@ _No input._ _No output._ +## dagger.#Service + +A reference to a network service endpoint, for example: - A TCP or UDP port - A unix or npipe socket - An HTTPS endpoint + +### dagger.#Service Inputs + +_No input._ + +### dagger.#Service Outputs + +_No output._ + ## dagger.#Stream Dagger stream. Can be mounted as a UNIX socket. diff --git a/docs/reference/git.md b/docs/reference/git.md index 59990600..bb0d9bfa 100644 --- a/docs/reference/git.md +++ b/docs/reference/git.md @@ -19,7 +19,7 @@ Commit & push to git repository | Name | Type | Description | | ------------- |:-------------: |:-------------: | |*repository.remote* | `string` |Repository remote URL | -|*repository.authToken* | `dagger.#Secret` |Authentication token (PAT or password) | +|*repository.authToken* | `*null \| {}` |Authentication token (PAT or password) | |*repository.branch* | `string` |Git branch | |*name* | `string` |Username | |*email* | `string` |Email | @@ -69,8 +69,8 @@ A git repository |*remote* | `string` |Git remote link | |*ref* | `string` |Git ref: can be a commit, tag or branch. Example: "main" | |*subdir* | `*null \| string` |(optional) Subdirectory | -|*authToken* | `dagger.#Secret` |(optional) Add Personal Access Token | -|*authHeader* | `dagger.#Secret` |(optional) Add OAuth Token | +|*authToken* | `*null \| {}` |(optional) Add Personal Access Token | +|*authHeader* | `*null \| {}` |(optional) Add OAuth Token | ### git.#Repository Outputs diff --git a/docs/reference/kubernetes/README.md b/docs/reference/kubernetes/README.md index 0e702977..e09fbfc6 100644 --- a/docs/reference/kubernetes/README.md +++ b/docs/reference/kubernetes/README.md @@ -37,7 +37,7 @@ Apply Kubernetes resources |*url* | `*null \| string` |Kubernetes manifest url to deploy remote configuration | |*namespace* | `*"default" \| string` |Kubernetes Namespace to deploy to | |*version* | `*"v1.19.9" \| string` |Version of kubectl client | -|*kubeconfig* | `dagger.#Secret` |Kube config file | +|*kubeconfig* | `(string\|struct)` |Kube config file | ### kubernetes.#Resources Outputs diff --git a/docs/reference/kubernetes/helm.md b/docs/reference/kubernetes/helm.md index f17e5e56..dab0ea92 100644 --- a/docs/reference/kubernetes/helm.md +++ b/docs/reference/kubernetes/helm.md @@ -27,7 +27,7 @@ Install a Helm chart |*timeout* | `*"5m" \| string` |time to wait for any individual Kubernetes operation (like Jobs for hooks) | |*wait* | `*true \| bool` |if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as timeout | |*atomic* | `*true \| bool` |if set, installation process purges chart on fail. The wait option will be set automatically if atomic is used | -|*kubeconfig* | `dagger.#Secret` |Kube config file | +|*kubeconfig* | `(string\|struct)` |Kube config file | |*version* | `*"3.5.2" \| string` |Helm version | |*kubectlVersion* | `*"v1.19.9" \| string` |Kubectl version | diff --git a/environment/pipeline.go b/environment/pipeline.go index e1155891..65647097 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -270,16 +270,11 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) ( // dagger.#FS forward compat // FIXME: remove this if isFS(op) { - id, err := op.LookupPath(fsIDPath).String() + fs, err := p.pctx.FS.FromValue(op) if err != nil { - return st, err + return st, nil } - - fs := p.pctx.FS.Get(plancontext.ContextKey(id)) - if fs == nil { - return st, fmt.Errorf("fs %q not found", id) - } - return fs.Result.ToState() + return fs.Result().ToState() } do, err := op.Lookup("do").String() @@ -595,30 +590,26 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value) // eg. mount: "/foo": secret: mysecret if secret := mnt.Lookup("secret"); secret.Exists() { - id, err := getSecretID(secret) + s, err := p.pctx.Secrets.FromValue(secret) if err != nil { return nil, err } return llb.AddSecret(dest, - llb.SecretID(id), + llb.SecretID(s.ID()), llb.SecretFileOpt(0, 0, 0400), // uid, gid, mask) ), nil } // eg. mount: "/var/run/docker.sock": stream: mystream if stream := mnt.Lookup("stream"); stream.Exists() { - if !stream.HasAttr("stream") { - return nil, fmt.Errorf("invalid stream %q: not a stream", stream.Path().String()) - } - - id, err := stream.Lookup("id").String() + s, err := p.pctx.Services.FromValue(stream) if err != nil { - return nil, fmt.Errorf("invalid stream %q: %w", stream.Path().String(), err) + return nil, err } return llb.AddSSHSocket( - llb.SSHID(id), + llb.SSHID(s.ID()), llb.SSHSocketTarget(dest), ), nil } @@ -761,15 +752,11 @@ func parseStringOrSecret(pctx *plancontext.Context, v *compiler.Value) (string, } // If we get here, it's a secret - id, err := getSecretID(v) + secret, err := pctx.Secrets.FromValue(v) if err != nil { return "", err } - secret := pctx.Secrets.Get(plancontext.ContextKey(id)) - if secret == nil { - return "", fmt.Errorf("secret %s not found", id) - } - return secret.PlainText, nil + return secret.PlainText(), nil } func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { @@ -973,21 +960,6 @@ func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.Sta ), nil } -func getSecretID(secretField *compiler.Value) (string, error) { - if !secretField.HasAttr("secret") { - return "", fmt.Errorf("invalid secret %q: not a secret", secretField.Path().String()) - } - idValue := secretField.Lookup("id") - if !idValue.Exists() { - return "", fmt.Errorf("invalid secret %q: no id field", secretField.Path().String()) - } - id, err := idValue.String() - if err != nil { - return "", fmt.Errorf("invalid secret id: %w", err) - } - return id, nil -} - func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { remote, err := op.Lookup("remote").String() if err != nil { @@ -1017,18 +989,18 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.Stat } // Secret if authToken := op.Lookup("authToken"); authToken.Exists() { - id, err := getSecretID(authToken) + authTokenSecret, err := p.pctx.Secrets.FromValue(authToken) if err != nil { return st, err } - gitOpts = append(gitOpts, llb.AuthTokenSecret(id)) + gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID())) } if authHeader := op.Lookup("authHeader"); authHeader.Exists() { - id, err := getSecretID(authHeader) + authHeaderSecret, err := p.pctx.Secrets.FromValue(authHeader) if err != nil { return st, err } - gitOpts = append(gitOpts, llb.AuthHeaderSecret(id)) + gitOpts = append(gitOpts, llb.AuthHeaderSecret(authHeaderSecret.ID())) } gitOpts = append(gitOpts, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remoteRedacted, ref))) diff --git a/plan/plan.go b/plan/plan.go index f41364c4..d6802fbb 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -67,10 +67,7 @@ func (p *Plan) registerLocalDirs() error { if err != nil { return err } - - p.context.LocalDirs.Register(&plancontext.LocalDir{ - Path: dir, - }) + p.context.LocalDirs.Add(dir) } return nil diff --git a/plan/task/import.go b/plan/task/import.go index 5f54e5e6..20feb38c 100644 --- a/plan/task/import.go +++ b/plan/task/import.go @@ -68,13 +68,10 @@ func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver return nil, err } - id := pctx.FS.Register(&plancontext.FS{ - Result: result, - }) - - return compiler.NewValueWithContent(id, - cue.Str("fs"), - cue.Hid("_fs", "alpha.dagger.io/dagger"), - cue.Str("id"), - ) + fs := pctx.FS.New(result) + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("fs"), fs.Value()); err != nil { + return nil, err + } + return out, nil } diff --git a/plan/task/secretenv.go b/plan/task/secretenv.go index 6ec3c821..90fc0b6d 100644 --- a/plan/task/secretenv.go +++ b/plan/task/secretenv.go @@ -36,12 +36,10 @@ func (c secretEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ sol if env == "" { return nil, fmt.Errorf("environment variable %q not set", secretEnv.Envvar) } - id := pctx.Secrets.Register(&plancontext.Secret{ - PlainText: env, - }) - - return compiler.NewValueWithContent(id, - cue.Str("contents"), - cue.Str("id"), - ) + secret := pctx.Secrets.New(env) + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("contents"), secret.Value()); err != nil { + return nil, err + } + return out, nil } diff --git a/plan/task/secretfile.go b/plan/task/secretfile.go index e75d056d..2aba3eb0 100644 --- a/plan/task/secretfile.go +++ b/plan/task/secretfile.go @@ -35,12 +35,11 @@ func (c secretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ so if err != nil { return nil, err } - id := pctx.Secrets.Register(&plancontext.Secret{ - PlainText: string(data), - }) - return compiler.NewValueWithContent(id, - cue.Str("contents"), - cue.Str("id"), - ) + secret := pctx.Secrets.New(string(data)) + out := compiler.NewValue() + if err := out.FillPath(cue.ParsePath("contents"), secret.Value()); err != nil { + return nil, err + } + return out, nil } diff --git a/plancontext/context.go b/plancontext/context.go index 2efa74cf..8b4fc548 100644 --- a/plancontext/context.go +++ b/plancontext/context.go @@ -2,18 +2,10 @@ package plancontext import ( "crypto/sha256" - "encoding/json" "fmt" ) -type ContextKey string - // Context holds the execution context for a plan. -// -// Usage: -// ctx := plancontext.New() -// id := ctx.Secrets.Register("mysecret") -// secret := ctx.Secrets.Get(id) type Context struct { Platform *platformContext FS *fsContext @@ -28,25 +20,26 @@ func New() *Context { platform: defaultPlatform, }, FS: &fsContext{ - store: make(map[ContextKey]*FS), + store: make(map[string]*FS), }, LocalDirs: &localDirContext{ - store: make(map[ContextKey]*LocalDir), + store: []string{}, }, Secrets: &secretContext{ - store: make(map[ContextKey]*Secret), + store: make(map[string]*Secret), }, Services: &serviceContext{ - store: make(map[ContextKey]*Service), + store: make(map[string]*Service), }, } } -func hashID(v interface{}) ContextKey { - data, err := json.Marshal(v) - if err != nil { - panic(err) +func hashID(values ...string) string { + hash := sha256.New() + for _, v := range values { + if _, err := hash.Write([]byte(v)); err != nil { + panic(err) + } } - hash := sha256.Sum256(data) - return ContextKey(fmt.Sprintf("%x", hash)) + return fmt.Sprintf("%x", hash) } diff --git a/plancontext/context_test.go b/plancontext/context_test.go index 652c22fa..bbd3e300 100644 --- a/plancontext/context_test.go +++ b/plancontext/context_test.go @@ -9,10 +9,8 @@ import ( func TestContext(t *testing.T) { ctx := New() - id := ctx.Secrets.Register(&Secret{ - PlainText: "test", - }) - secret := ctx.Secrets.Get(id) - require.NotNil(t, secret) - require.Equal(t, "test", secret.PlainText) + secret := ctx.Secrets.New("test") + get, err := ctx.Secrets.FromValue(secret.Value()) + require.NoError(t, err) + require.Equal(t, "test", get.PlainText()) } diff --git a/plancontext/fs.go b/plancontext/fs.go index 7a8b1267..8073b95c 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -1,32 +1,71 @@ package plancontext import ( + "fmt" "sync" + "cuelang.org/go/cue" + "github.com/google/uuid" bkgw "github.com/moby/buildkit/frontend/gateway/client" + "go.dagger.io/dagger/compiler" +) + +var ( + fsIDPath = cue.MakePath( + cue.Hid("_fs", "alpha.dagger.io/dagger"), + cue.Str("id"), + ) ) type FS struct { - Result bkgw.Reference + id string + result bkgw.Reference +} + +func (fs *FS) Result() bkgw.Reference { + return fs.result +} + +func (fs *FS) Value() *compiler.Value { + v := compiler.NewValue() + if err := v.FillPath(fsIDPath, fs.id); err != nil { + panic(err) + } + return v } type fsContext struct { l sync.RWMutex - store map[ContextKey]*FS + store map[string]*FS } -func (c *fsContext) Register(fs *FS) ContextKey { +func (c *fsContext) New(result bkgw.Reference) *FS { c.l.Lock() defer c.l.Unlock() - id := hashID(fs) - c.store[id] = fs - return id + fs := &FS{ + // FIXME: get a hash from result instead + id: uuid.New().String(), + result: result, + } + + c.store[fs.id] = fs + return fs } -func (c *fsContext) Get(id ContextKey) *FS { +func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) { c.l.RLock() defer c.l.RUnlock() - return c.store[id] + id, err := v.LookupPath(fsIDPath).String() + if err != nil { + return nil, fmt.Errorf("invalid FS %q: %w", v.Path(), err) + } + + fs, ok := c.store[id] + if !ok { + return nil, fmt.Errorf("fs %q not found", id) + } + + return fs, nil } diff --git a/plancontext/localdir.go b/plancontext/localdir.go index b2ea9494..8600cb61 100644 --- a/plancontext/localdir.go +++ b/plancontext/localdir.go @@ -5,41 +5,16 @@ import ( "sync" ) -type LocalDir struct { - Path string -} - type localDirContext struct { l sync.RWMutex - store map[ContextKey]*LocalDir + store []string } -func (c *localDirContext) Register(directory *LocalDir) ContextKey { +func (c *localDirContext) Add(dir string) { c.l.Lock() defer c.l.Unlock() - id := hashID(directory) - c.store[id] = directory - return id -} - -func (c *localDirContext) Get(id ContextKey) *LocalDir { - c.l.RLock() - defer c.l.RUnlock() - - return c.store[id] -} - -func (c *localDirContext) List() []*LocalDir { - c.l.RLock() - defer c.l.RUnlock() - - directories := make([]*LocalDir, 0, len(c.store)) - for _, d := range c.store { - directories = append(directories, d) - } - - return directories + c.store = append(c.store, dir) } func (c *localDirContext) Paths() (map[string]string, error) { @@ -48,12 +23,12 @@ func (c *localDirContext) Paths() (map[string]string, error) { directories := make(map[string]string) for _, d := range c.store { - abs, err := filepath.Abs(d.Path) + abs, err := filepath.Abs(d) if err != nil { return nil, err } - directories[d.Path] = abs + directories[d] = abs } return directories, nil diff --git a/plancontext/secret.go b/plancontext/secret.go index 2c052d69..6ccd75c1 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -1,26 +1,77 @@ package plancontext -import "sync" +import ( + "fmt" + "sync" + + "cuelang.org/go/cue" + "go.dagger.io/dagger/compiler" +) + +var ( + secretIDPath = cue.MakePath( + cue.Hid("_secret", "alpha.dagger.io/dagger"), + cue.Str("id"), + ) +) + +type Secret struct { + id string + plainText string +} + +func (s *Secret) ID() string { + return s.id +} + +func (s *Secret) PlainText() string { + return s.plainText +} + +func (s *Secret) Value() *compiler.Value { + v := compiler.NewValue() + if err := v.FillPath(secretIDPath, s.id); err != nil { + panic(err) + } + return v +} type secretContext struct { l sync.RWMutex - store map[ContextKey]*Secret + store map[string]*Secret } -type Secret struct { - PlainText string -} +func (c *secretContext) New(plaintext string) *Secret { + secret := &Secret{ + id: hashID(plaintext), + plainText: plaintext, + } -func (c *secretContext) Register(secret *Secret) ContextKey { c.l.Lock() defer c.l.Unlock() - id := hashID(secret.PlainText) - c.store[id] = secret - return id + c.store[secret.id] = secret + return secret } -func (c *secretContext) Get(id ContextKey) *Secret { +func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) { + c.l.RLock() + defer c.l.RUnlock() + + id, err := v.LookupPath(secretIDPath).String() + if err != nil { + return nil, fmt.Errorf("invalid secret %q: %w", v.Path(), err) + } + + secret, ok := c.store[id] + if !ok { + return nil, fmt.Errorf("secret %q not found", id) + } + + return secret, nil +} + +func (c *secretContext) Get(id string) *Secret { c.l.RLock() defer c.l.RUnlock() diff --git a/plancontext/service.go b/plancontext/service.go index 3dbffdc2..3f5ad71f 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -1,27 +1,84 @@ package plancontext -import "sync" +import ( + "fmt" + "sync" + + "cuelang.org/go/cue" + "go.dagger.io/dagger/compiler" +) + +var ( + serviceIDPath = cue.MakePath( + cue.Hid("_service", "alpha.dagger.io/dagger"), + cue.Str("id"), + ) +) + +type Service struct { + id string + + unix string + npipe string +} + +func (s *Service) ID() string { + return s.id +} + +func (s *Service) Unix() string { + return s.unix +} + +func (s *Service) NPipe() string { + return s.npipe +} + +func (s *Service) Value() *compiler.Value { + v := compiler.NewValue() + if err := v.FillPath(serviceIDPath, s.id); err != nil { + panic(err) + } + return v +} type serviceContext struct { l sync.RWMutex - store map[ContextKey]*Service + store map[string]*Service } -type Service struct { - Unix string - Npipe string -} - -func (c *serviceContext) Register(service *Service) ContextKey { +func (c *serviceContext) New(unix, npipe string) *Service { c.l.Lock() defer c.l.Unlock() - id := hashID(service) - c.store[id] = service - return id + s := &Service{ + id: hashID(unix, npipe), + unix: unix, + npipe: npipe, + } + + c.store[s.id] = s + return s } -func (c *serviceContext) Get(id ContextKey) *Service { +func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) { + c.l.RLock() + defer c.l.RUnlock() + + id, err := v.LookupPath(serviceIDPath).String() + if err != nil { + return nil, fmt.Errorf("invalid service %q: %w", v.Path(), err) + } + + s, ok := c.store[id] + if !ok { + return nil, fmt.Errorf("service %q not found", id) + } + + return s, nil +} + +func (c *serviceContext) Get(id string) *Service { c.l.RLock() defer c.l.RUnlock() diff --git a/solver/secretsprovider.go b/solver/secretsprovider.go index 8e811bde..664f70a3 100644 --- a/solver/secretsprovider.go +++ b/solver/secretsprovider.go @@ -21,7 +21,7 @@ type inputStore struct { func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) { lg := log.Ctx(ctx) - secret := s.pctx.Secrets.Get(plancontext.ContextKey(id)) + secret := s.pctx.Secrets.Get(id) if secret == nil { return nil, secrets.ErrNotFound } @@ -31,5 +31,5 @@ func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) { Str("id", id). Msg("injecting secret") - return []byte(secret.PlainText), nil + return []byte(secret.PlainText()), nil } diff --git a/solver/socketprovider.go b/solver/socketprovider.go index 00d9ec94..25358094 100644 --- a/solver/socketprovider.go +++ b/solver/socketprovider.go @@ -36,7 +36,7 @@ func (sp *SocketProvider) ForwardAgent(stream sshforward.SSH_ForwardAgentServer) id = v[0] } - service := sp.pctx.Services.Get(plancontext.ContextKey(id)) + service := sp.pctx.Services.Get(id) if service == nil { return fmt.Errorf("invalid socket id %q", id) } diff --git a/solver/socketprovider_unix.go b/solver/socketprovider_unix.go index 529ed0ea..fb83ce1b 100644 --- a/solver/socketprovider_unix.go +++ b/solver/socketprovider_unix.go @@ -12,9 +12,9 @@ import ( ) func dialService(service *plancontext.Service) (net.Conn, error) { - if service.Unix == "" { + if service.Unix() == "" { return nil, errors.New("unsupported socket type") } - return net.DialTimeout("unix", service.Unix, time.Second) + return net.DialTimeout("unix", service.Unix(), time.Second) } diff --git a/solver/socketprovider_windows.go b/solver/socketprovider_windows.go index 0a425527..d643e5e9 100644 --- a/solver/socketprovider_windows.go +++ b/solver/socketprovider_windows.go @@ -13,10 +13,10 @@ import ( ) func dialService(service *plancontext.Service) (net.Conn, error) { - if service.Npipe == "" { + if service.NPipe() == "" { return nil, errors.New("unsupported socket type") } dur := time.Second - return winio.DialPipe(service.Npipe, &dur) + return winio.DialPipe(service.NPipe(), &dur) } diff --git a/state/input.go b/state/input.go index 48f561b2..22b1fefd 100644 --- a/state/input.go +++ b/state/input.go @@ -12,7 +12,6 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" - "go.dagger.io/dagger/plancontext" ) // An input is a value or artifact supplied by the user. @@ -121,9 +120,7 @@ func (dir dirInput) Compile(state *State) (*compiler.Value, error) { return nil, err } - state.Context.LocalDirs.Register(&plancontext.LocalDir{ - Path: p, - }) + state.Context.LocalDirs.Add(p) llb := fmt.Sprintf( `#up: [{do: "local", dir: %s, include: %s, exclude: %s}]`, @@ -195,11 +192,8 @@ func SecretInput(data string) Input { type secretInput string func (i secretInput) Compile(st *State) (*compiler.Value, error) { - id := st.Context.Secrets.Register(&plancontext.Secret{ - PlainText: i.PlainText(), - }) - secretValue := fmt.Sprintf(`{id: %q}`, id) - return compiler.Compile("", secretValue) + secret := st.Context.Secrets.New(i.PlainText()) + return secret.Value(), nil } func (i secretInput) PlainText() string { @@ -301,10 +295,6 @@ type socketInput struct { } func (i socketInput) Compile(st *State) (*compiler.Value, error) { - id := st.Context.Services.Register(&plancontext.Service{ - Unix: i.Unix, - Npipe: i.Npipe, - }) - socketValue := fmt.Sprintf(`{id: %q}`, id) - return compiler.Compile("", socketValue) + service := st.Context.Services.New(i.Unix, i.Npipe) + return service.Value(), nil } diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index c37f6621..8dd631cd 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -24,13 +24,18 @@ import ( ... } -// Dagger stream. Can be mounted as a UNIX socket. -#Stream: { - @dagger(stream) - - id: string +// A reference to a network service endpoint, for example: +// - A TCP or UDP port +// - A unix or npipe socket +// - An HTTPS endpoint +#Service: { + _service: id: string } +// Dagger stream. Can be mounted as a UNIX socket. +// FIXME: Deprecated. For backward compatibility only, use #Service instead. +#Stream: #Service + // A reference to an external secret, for example: // - A password // - A SSH private key @@ -38,9 +43,7 @@ import ( // Secrets are never merged in the Cue tree. They can only be used // by a special filesystem mount designed to minimize leak risk. #Secret: { - @dagger(secret) - - id: string + _secret: id: string } #Input: { diff --git a/tests/compute/secrets/simple/simple.cue b/tests/compute/secrets/simple/simple.cue index 0b8d3d7f..43374444 100644 --- a/tests/compute/secrets/simple/simple.cue +++ b/tests/compute/secrets/simple/simple.cue @@ -17,7 +17,6 @@ TestSecrets: #up: [ op.#Exec & { mount: "/secret": secret: mySecret - env: PLAIN: mySecret.id args: [ "/bin/bash", "--noprofile", @@ -27,7 +26,6 @@ TestSecrets: #up: [ "-c", #""" test "$(cat /secret)" = "SecretValue" - test "$PLAIN" != "SecretValue" """#, ] }, diff --git a/tests/core.bats b/tests/core.bats index 25fa80f2..04e5c88d 100644 --- a/tests/core.bats +++ b/tests/core.bats @@ -141,10 +141,6 @@ setup() { "$DAGGER" input secret mySecret SecretValue run "$DAGGER" up assert_success - - # Make sure the secret doesn't show in dagger query - run "$DAGGER" query mySecret.id -f text - assert_success } @test "core: stream" { From 337ed0d1cc26660c9af6c1f1a6a1d51fe591df6b Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 1 Dec 2021 12:03:41 -0800 Subject: [PATCH 2/7] remove hardcoded stdlib path Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/common/common.go | 3 ++- cmd/dagger/cmd/doc.go | 2 +- mod/mod.go | 3 ++- mod/require.go | 8 +++++--- plancontext/fs.go | 3 ++- plancontext/secret.go | 3 ++- plancontext/service.go | 3 ++- state/project.go | 4 ++-- stdlib/stdlib.go | 5 +++-- 9 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index bb875234..71929ccd 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -12,6 +12,7 @@ import ( "go.dagger.io/dagger/client" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/state" + "go.dagger.io/dagger/stdlib" ) func CurrentProject(ctx context.Context) *state.Project { @@ -90,7 +91,7 @@ func FormatValue(val *compiler.Value) string { } if val.LookupPath(cue.MakePath( - cue.Hid("_secret", "alpha.dagger.io/dagger"), + cue.Hid("_secret", stdlib.PackageName), cue.Str("id"), )).Exists() { return "dagger.#Secret" diff --git a/cmd/dagger/cmd/doc.go b/cmd/dagger/cmd/doc.go index 049536e2..dcdc3f3f 100644 --- a/cmd/dagger/cmd/doc.go +++ b/cmd/dagger/cmd/doc.go @@ -346,7 +346,7 @@ func walkStdlib(ctx context.Context, output, format string) { return nil } - pkgName := fmt.Sprintf("alpha.dagger.io/%s", p) + pkgName := fmt.Sprintf("%s/%s", stdlib.ModuleName, p) lg.Info().Str("package", pkgName).Str("format", format).Msg("generating doc") val, err := loadCode(pkgName) if err != nil { diff --git a/mod/mod.go b/mod/mod.go index bf49c284..21bfd0d1 100644 --- a/mod/mod.go +++ b/mod/mod.go @@ -8,6 +8,7 @@ import ( "github.com/gofrs/flock" "github.com/rs/zerolog/log" + "go.dagger.io/dagger/stdlib" ) const ( @@ -15,7 +16,7 @@ const ( ) func isUniverse(repoName string) bool { - return strings.HasPrefix(strings.ToLower(repoName), "alpha.dagger.io") + return strings.HasPrefix(strings.ToLower(repoName), stdlib.ModuleName) } func Install(ctx context.Context, workspace, repoName, versionConstraint string) (*Require, error) { diff --git a/mod/require.go b/mod/require.go index 157a7f1a..4e04ee07 100644 --- a/mod/require.go +++ b/mod/require.go @@ -7,6 +7,8 @@ import ( "path/filepath" "regexp" "strings" + + "go.dagger.io/dagger/stdlib" ) type Require struct { @@ -25,7 +27,7 @@ func newRequire(repoName, versionConstraint string) (*Require, error) { switch { case strings.HasPrefix(repoName, "github.com"): return parseGithubRepoName(repoName, versionConstraint) - case strings.HasPrefix(repoName, "alpha.dagger.io"): + case strings.HasPrefix(repoName, stdlib.ModuleName): return parseDaggerRepoName(repoName, versionConstraint) default: return nil, fmt.Errorf("repo name does not match suported providers") @@ -52,7 +54,7 @@ func parseGithubRepoName(repoName, versionConstraint string) (*Require, error) { }, nil } -var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) +var daggerRepoNameRegex = regexp.MustCompile(stdlib.ModuleName + `([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) func parseDaggerRepoName(repoName, versionConstraint string) (*Require, error) { repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName) @@ -62,7 +64,7 @@ func parseDaggerRepoName(repoName, versionConstraint string) (*Require, error) { } return &Require{ - repo: "alpha.dagger.io", + repo: stdlib.ModuleName, path: repoMatches[1], version: repoMatches[2], versionConstraint: versionConstraint, diff --git a/plancontext/fs.go b/plancontext/fs.go index 8073b95c..f0b012f2 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -8,11 +8,12 @@ import ( "github.com/google/uuid" bkgw "github.com/moby/buildkit/frontend/gateway/client" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/stdlib" ) var ( fsIDPath = cue.MakePath( - cue.Hid("_fs", "alpha.dagger.io/dagger"), + cue.Hid("_fs", stdlib.PackageName), cue.Str("id"), ) ) diff --git a/plancontext/secret.go b/plancontext/secret.go index 6ccd75c1..a03b2739 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -6,11 +6,12 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/stdlib" ) var ( secretIDPath = cue.MakePath( - cue.Hid("_secret", "alpha.dagger.io/dagger"), + cue.Hid("_secret", stdlib.PackageName), cue.Str("id"), ) ) diff --git a/plancontext/service.go b/plancontext/service.go index 3f5ad71f..a8079099 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -6,11 +6,12 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/stdlib" ) var ( serviceIDPath = cue.MakePath( - cue.Hid("_service", "alpha.dagger.io/dagger"), + cue.Hid("_service", stdlib.PackageName), cue.Str("id"), ) ) diff --git a/state/project.go b/state/project.go index db99b71d..ec180a8f 100644 --- a/state/project.go +++ b/state/project.go @@ -399,7 +399,7 @@ func VendorUniverse(ctx context.Context, p string) error { // add universe and lock file to `.gitignore` if err := os.WriteFile( path.Join(p, "cue.mod", "pkg", ".gitignore"), - []byte(fmt.Sprintf("# generated by dagger\n%s\ndagger.lock\n", stdlib.PackageName)), + []byte(fmt.Sprintf("# generated by dagger\n%s\ndagger.lock\n", stdlib.ModuleName)), 0600, ); err != nil { return err @@ -408,7 +408,7 @@ func VendorUniverse(ctx context.Context, p string) error { log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe") if err := stdlib.Vendor(ctx, p); err != nil { // FIXME(samalba): disabled install remote stdlib temporarily - // if _, err := mod.Install(ctx, p, "alpha.dagger.io", ""); err != nil { + // if _, err := mod.Install(ctx, p, stdlib.ModuleName, ""); err != nil { return err } diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index a31ce6cc..0c255882 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -17,8 +17,9 @@ var ( //go:embed **/*.cue **/*/*.cue FS embed.FS - PackageName = "alpha.dagger.io" - Path = path.Join("cue.mod", "pkg", PackageName) + ModuleName = "alpha.dagger.io" + PackageName = fmt.Sprintf("%s/dagger", ModuleName) + Path = path.Join("cue.mod", "pkg", ModuleName) lockFilePath = path.Join("cue.mod", "dagger.lock") ) From a25e6bb3a2b796b5c82ae9bf23eadcfc4991cd86 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 1 Dec 2021 17:56:31 -0800 Subject: [PATCH 3/7] include hidden fields in concreteness check Signed-off-by: Andrea Luzzardi --- compiler/value.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/value.go b/compiler/value.go index cfcddbc0..372070ad 100644 --- a/compiler/value.go +++ b/compiler/value.go @@ -156,7 +156,10 @@ func (v *Value) IsConcrete() bool { // Recursive concreteness check. func (v *Value) IsConcreteR(opts ...cue.Option) error { - o := []cue.Option{cue.Concrete(true)} + o := []cue.Option{ + cue.Concrete(true), + cue.Hidden(true), + } o = append(o, opts...) return v.val.Validate(o...) } From 6aa4f6b8b3ce54e508abb25305eed637a2da72cc Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 7 Dec 2021 14:02:17 -0500 Subject: [PATCH 4/7] secretfile: data -> plaintext Signed-off-by: Andrea Luzzardi --- plan/task/secretfile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plan/task/secretfile.go b/plan/task/secretfile.go index 2aba3eb0..344b0f96 100644 --- a/plan/task/secretfile.go +++ b/plan/task/secretfile.go @@ -31,12 +31,12 @@ func (c secretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ so lg.Debug().Str("path", secretFile.Path).Msg("loading secret") - data, err := os.ReadFile(secretFile.Path) + plaintext, err := os.ReadFile(secretFile.Path) if err != nil { return nil, err } - secret := pctx.Secrets.New(string(data)) + secret := pctx.Secrets.New(string(plaintext)) out := compiler.NewValue() if err := out.FillPath(cue.ParsePath("contents"), secret.Value()); err != nil { return nil, err From 201ac391b49b515364f4f44d0402bc5be65b8b6e Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 7 Dec 2021 14:03:07 -0500 Subject: [PATCH 5/7] check for concreteness in specialized types Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/input/list.go | 15 +++++++ cmd/dagger/cmd/up.go | 18 +++++++- compiler/value.go | 1 + environment/pipeline.go | 81 ++++++------------------------------ plancontext/fs.go | 4 ++ plancontext/secret.go | 4 ++ plancontext/service.go | 4 ++ 7 files changed, 57 insertions(+), 70 deletions(-) diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index e66aac28..dd7cc07d 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -60,6 +60,21 @@ var listCmd = &cobra.Command{ isConcrete := (inp.IsConcreteR() == nil) _, hasDefault := inp.Default() + switch { + case env.Context().Secrets.Contains(inp): + if _, err := env.Context().Secrets.FromValue(inp); err != nil { + isConcrete = false + } + case env.Context().FS.Contains(inp): + if _, err := env.Context().FS.FromValue(inp); err != nil { + isConcrete = false + } + case env.Context().Services.Contains(inp): + if _, err := env.Context().Services.FromValue(inp); err != nil { + isConcrete = false + } + } + if !viper.GetBool("all") { // skip input that is not overridable if !hasDefault && isConcrete { diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 519e39b8..1dc859b7 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -140,7 +140,23 @@ func checkInputs(ctx context.Context, env *environment.Environment) error { } for _, i := range inputs { - if i.IsConcreteR(cue.Optional(true)) != nil { + isConcrete := (i.IsConcreteR(cue.Optional(true)) == nil) + switch { + case env.Context().Secrets.Contains(i): + if _, err := env.Context().Secrets.FromValue(i); err != nil { + isConcrete = false + } + case env.Context().FS.Contains(i): + if _, err := env.Context().FS.FromValue(i); err != nil { + isConcrete = false + } + case env.Context().Services.Contains(i): + if _, err := env.Context().Services.FromValue(i); err != nil { + isConcrete = false + } + } + + if !isConcrete { notConcreteInputs = append(notConcreteInputs, i) } } diff --git a/compiler/value.go b/compiler/value.go index 372070ad..30fdac18 100644 --- a/compiler/value.go +++ b/compiler/value.go @@ -157,6 +157,7 @@ func (v *Value) IsConcrete() bool { // Recursive concreteness check. func (v *Value) IsConcreteR(opts ...cue.Option) error { o := []cue.Option{ + cue.All(), cue.Concrete(true), cue.Hidden(true), } diff --git a/environment/pipeline.go b/environment/pipeline.go index 65647097..76beef79 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -40,13 +40,6 @@ const ( StateCompleted = State("completed") ) -var ( - fsIDPath = cue.MakePath( - cue.Hid("_fs", "alpha.dagger.io/dagger"), - cue.Str("id"), - ) -) - // An execution pipeline type Pipeline struct { code *compiler.Value @@ -102,90 +95,40 @@ func IsComponent(v *compiler.Value) bool { return v.Lookup("#up").Exists() } -func isFS(v *compiler.Value) bool { - return v.LookupPath(fsIDPath).Exists() -} - -func ops(code *compiler.Value) ([]*compiler.Value, error) { +func (p *Pipeline) ops() ([]*compiler.Value, error) { ops := []*compiler.Value{} // dagger.#FS forward compat // FIXME: remove this - if isFS(code) { - ops = append(ops, code) + if p.pctx.FS.Contains(p.code) { + ops = append(ops, p.code) } // 1. attachment array - if IsComponent(code) { - xops, err := code.Lookup("#up").List() + if IsComponent(p.code) { + xops, err := p.code.Lookup("#up").List() if err != nil { return nil, err } // 'from' has an executable attached ops = append(ops, xops...) // 2. individual op - } else if _, err := code.Lookup("do").String(); err == nil { - ops = append(ops, code) + } else if _, err := p.code.Lookup("do").String(); err == nil { + ops = append(ops, p.code) // 3. op array - } else if xops, err := code.List(); err == nil { + } else if xops, err := p.code.List(); err == nil { ops = append(ops, xops...) } else { // 4. error - source, err := code.Source() + source, err := p.code.Source() if err != nil { panic(err) } - return nil, fmt.Errorf("not executable: %s (%s)", source, code.Path().String()) + return nil, fmt.Errorf("not executable: %s (%s)", source, p.code.Path().String()) } return ops, nil } -func Analyze(fn func(*compiler.Value) error, code *compiler.Value) error { - ops, err := ops(code) - if err != nil { - // Ignore CUE errors when analyzing. This might be because the value is - // not concrete since static analysis runs before pipelines are executed. - return nil - } - for _, op := range ops { - if err := analyzeOp(fn, op); err != nil { - return err - } - } - return nil -} - -func analyzeOp(fn func(*compiler.Value) error, op *compiler.Value) error { - // dagger.#FS forward compat - // FIXME: remove this - if isFS(op) { - return nil - } - - if err := fn(op); err != nil { - return err - } - do, err := op.Lookup("do").String() - if err != nil { - return err - } - switch do { - case "load", "copy": - return Analyze(fn, op.Lookup("from")) - case "exec": - fields, err := op.Lookup("mount").Fields() - if err != nil { - return err - } - for _, mnt := range fields { - if from := mnt.Value.Lookup("from"); from.Exists() { - return Analyze(fn, from) - } - } - } - return nil -} - func (p *Pipeline) Run(ctx context.Context) error { lg := log. Ctx(ctx). @@ -230,7 +173,7 @@ func (p *Pipeline) Run(ctx context.Context) error { } func (p *Pipeline) run(ctx context.Context) error { - ops, err := ops(p.code) + ops, err := p.ops() if err != nil { return err } @@ -269,7 +212,7 @@ func (p *Pipeline) run(ctx context.Context) error { func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { // dagger.#FS forward compat // FIXME: remove this - if isFS(op) { + if p.pctx.FS.Contains(op) { fs, err := p.pctx.FS.FromValue(op) if err != nil { return st, nil diff --git a/plancontext/fs.go b/plancontext/fs.go index f0b012f2..208a9827 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -54,6 +54,10 @@ func (c *fsContext) New(result bkgw.Reference) *FS { return fs } +func (c *fsContext) Contains(v *compiler.Value) bool { + return v.LookupPath(fsIDPath).Exists() +} + func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) { c.l.RLock() defer c.l.RUnlock() diff --git a/plancontext/secret.go b/plancontext/secret.go index a03b2739..8ff97dd1 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -55,6 +55,10 @@ func (c *secretContext) New(plaintext string) *Secret { return secret } +func (c *secretContext) Contains(v *compiler.Value) bool { + return v.LookupPath(secretIDPath).Exists() +} + func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) { c.l.RLock() defer c.l.RUnlock() diff --git a/plancontext/service.go b/plancontext/service.go index a8079099..4b4c7f9c 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -62,6 +62,10 @@ func (c *serviceContext) New(unix, npipe string) *Service { return s } +func (c *serviceContext) Contains(v *compiler.Value) bool { + return v.LookupPath(serviceIDPath).Exists() +} + func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) { c.l.RLock() defer c.l.RUnlock() From c881bc2dba1e73115667e3f0223dec189c05b904 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 7 Dec 2021 14:16:14 -0500 Subject: [PATCH 6/7] plancontext: use helpers to determine if CUE value is secret/fs/service Signed-off-by: Andrea Luzzardi --- cmd/dagger/cmd/common/common.go | 17 +++++++++-------- cmd/dagger/cmd/input/list.go | 7 ++++--- cmd/dagger/cmd/up.go | 7 ++++--- environment/pipeline.go | 4 ++-- plancontext/fs.go | 8 ++++---- plancontext/secret.go | 8 ++++---- plancontext/service.go | 8 ++++---- 7 files changed, 31 insertions(+), 28 deletions(-) diff --git a/cmd/dagger/cmd/common/common.go b/cmd/dagger/cmd/common/common.go index 71929ccd..295b20ae 100644 --- a/cmd/dagger/cmd/common/common.go +++ b/cmd/dagger/cmd/common/common.go @@ -11,8 +11,8 @@ import ( "github.com/spf13/viper" "go.dagger.io/dagger/client" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/state" - "go.dagger.io/dagger/stdlib" ) func CurrentProject(ctx context.Context) *state.Project { @@ -86,16 +86,17 @@ func CurrentEnvironmentState(ctx context.Context, project *state.Project) *state // FormatValue returns the String representation of the cue value func FormatValue(val *compiler.Value) string { - if val.HasAttr("artifact") { + switch { + case val.HasAttr("artifact"): return "dagger.#Artifact" + case plancontext.IsSecretValue(val): + return "dagger.#Secret" + case plancontext.IsFSValue(val): + return "dagger.#FS" + case plancontext.IsServiceValue(val): + return "dagger.#Service" } - if val.LookupPath(cue.MakePath( - cue.Hid("_secret", stdlib.PackageName), - cue.Str("id"), - )).Exists() { - return "dagger.#Secret" - } if val.IsConcreteR() != nil { return val.IncompleteKind().String() } diff --git a/cmd/dagger/cmd/input/list.go b/cmd/dagger/cmd/input/list.go index dd7cc07d..8589db02 100644 --- a/cmd/dagger/cmd/input/list.go +++ b/cmd/dagger/cmd/input/list.go @@ -10,6 +10,7 @@ import ( "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" @@ -61,15 +62,15 @@ var listCmd = &cobra.Command{ _, hasDefault := inp.Default() switch { - case env.Context().Secrets.Contains(inp): + case plancontext.IsSecretValue(inp): if _, err := env.Context().Secrets.FromValue(inp); err != nil { isConcrete = false } - case env.Context().FS.Contains(inp): + case plancontext.IsFSValue(inp): if _, err := env.Context().FS.FromValue(inp); err != nil { isConcrete = false } - case env.Context().Services.Contains(inp): + case plancontext.IsServiceValue(inp): if _, err := env.Context().Services.FromValue(inp); err != nil { isConcrete = false } diff --git a/cmd/dagger/cmd/up.go b/cmd/dagger/cmd/up.go index 1dc859b7..3e6960cd 100644 --- a/cmd/dagger/cmd/up.go +++ b/cmd/dagger/cmd/up.go @@ -13,6 +13,7 @@ import ( "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" "go.dagger.io/dagger/plan" + "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" "golang.org/x/term" @@ -142,15 +143,15 @@ func checkInputs(ctx context.Context, env *environment.Environment) error { for _, i := range inputs { isConcrete := (i.IsConcreteR(cue.Optional(true)) == nil) switch { - case env.Context().Secrets.Contains(i): + case plancontext.IsSecretValue(i): if _, err := env.Context().Secrets.FromValue(i); err != nil { isConcrete = false } - case env.Context().FS.Contains(i): + case plancontext.IsFSValue(i): if _, err := env.Context().FS.FromValue(i); err != nil { isConcrete = false } - case env.Context().Services.Contains(i): + case plancontext.IsServiceValue(i): if _, err := env.Context().Services.FromValue(i); err != nil { isConcrete = false } diff --git a/environment/pipeline.go b/environment/pipeline.go index 76beef79..31b8e8e3 100644 --- a/environment/pipeline.go +++ b/environment/pipeline.go @@ -100,7 +100,7 @@ func (p *Pipeline) ops() ([]*compiler.Value, error) { // dagger.#FS forward compat // FIXME: remove this - if p.pctx.FS.Contains(p.code) { + if plancontext.IsFSValue(p.code) { ops = append(ops, p.code) } @@ -212,7 +212,7 @@ func (p *Pipeline) run(ctx context.Context) error { func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { // dagger.#FS forward compat // FIXME: remove this - if p.pctx.FS.Contains(op) { + if plancontext.IsFSValue(op) { fs, err := p.pctx.FS.FromValue(op) if err != nil { return st, nil diff --git a/plancontext/fs.go b/plancontext/fs.go index 208a9827..dc8ca6a2 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -18,6 +18,10 @@ var ( ) ) +func IsFSValue(v *compiler.Value) bool { + return v.LookupPath(fsIDPath).Exists() +} + type FS struct { id string result bkgw.Reference @@ -54,10 +58,6 @@ func (c *fsContext) New(result bkgw.Reference) *FS { return fs } -func (c *fsContext) Contains(v *compiler.Value) bool { - return v.LookupPath(fsIDPath).Exists() -} - func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) { c.l.RLock() defer c.l.RUnlock() diff --git a/plancontext/secret.go b/plancontext/secret.go index 8ff97dd1..0a08ab67 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -16,6 +16,10 @@ var ( ) ) +func IsSecretValue(v *compiler.Value) bool { + return v.LookupPath(secretIDPath).Exists() +} + type Secret struct { id string plainText string @@ -55,10 +59,6 @@ func (c *secretContext) New(plaintext string) *Secret { return secret } -func (c *secretContext) Contains(v *compiler.Value) bool { - return v.LookupPath(secretIDPath).Exists() -} - func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) { c.l.RLock() defer c.l.RUnlock() diff --git a/plancontext/service.go b/plancontext/service.go index 4b4c7f9c..0b2a7bee 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -16,6 +16,10 @@ var ( ) ) +func IsServiceValue(v *compiler.Value) bool { + return v.LookupPath(serviceIDPath).Exists() +} + type Service struct { id string @@ -62,10 +66,6 @@ func (c *serviceContext) New(unix, npipe string) *Service { return s } -func (c *serviceContext) Contains(v *compiler.Value) bool { - return v.LookupPath(serviceIDPath).Exists() -} - func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) { c.l.RLock() defer c.l.RUnlock() From a58ca16c3da52701c26077c72090bc244d024397 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 7 Dec 2021 14:20:08 -0500 Subject: [PATCH 7/7] plancontext cleanup Signed-off-by: Andrea Luzzardi --- plan/task/import.go | 2 +- plan/task/secretenv.go | 2 +- plan/task/secretfile.go | 2 +- plancontext/context_test.go | 2 +- plancontext/fs.go | 2 +- plancontext/secret.go | 2 +- plancontext/service.go | 2 +- state/input.go | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plan/task/import.go b/plan/task/import.go index 20feb38c..a567a223 100644 --- a/plan/task/import.go +++ b/plan/task/import.go @@ -70,7 +70,7 @@ func (c importTask) Run(ctx context.Context, pctx *plancontext.Context, s solver fs := pctx.FS.New(result) out := compiler.NewValue() - if err := out.FillPath(cue.ParsePath("fs"), fs.Value()); err != nil { + if err := out.FillPath(cue.ParsePath("fs"), fs.MarshalCUE()); err != nil { return nil, err } return out, nil diff --git a/plan/task/secretenv.go b/plan/task/secretenv.go index 90fc0b6d..f3db6923 100644 --- a/plan/task/secretenv.go +++ b/plan/task/secretenv.go @@ -38,7 +38,7 @@ func (c secretEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ sol } secret := pctx.Secrets.New(env) out := compiler.NewValue() - if err := out.FillPath(cue.ParsePath("contents"), secret.Value()); err != nil { + if err := out.FillPath(cue.ParsePath("contents"), secret.MarshalCUE()); err != nil { return nil, err } return out, nil diff --git a/plan/task/secretfile.go b/plan/task/secretfile.go index 344b0f96..3cb5a335 100644 --- a/plan/task/secretfile.go +++ b/plan/task/secretfile.go @@ -38,7 +38,7 @@ func (c secretFileTask) Run(ctx context.Context, pctx *plancontext.Context, _ so secret := pctx.Secrets.New(string(plaintext)) out := compiler.NewValue() - if err := out.FillPath(cue.ParsePath("contents"), secret.Value()); err != nil { + if err := out.FillPath(cue.ParsePath("contents"), secret.MarshalCUE()); err != nil { return nil, err } return out, nil diff --git a/plancontext/context_test.go b/plancontext/context_test.go index bbd3e300..4cf788f1 100644 --- a/plancontext/context_test.go +++ b/plancontext/context_test.go @@ -10,7 +10,7 @@ func TestContext(t *testing.T) { ctx := New() secret := ctx.Secrets.New("test") - get, err := ctx.Secrets.FromValue(secret.Value()) + get, err := ctx.Secrets.FromValue(secret.MarshalCUE()) require.NoError(t, err) require.Equal(t, "test", get.PlainText()) } diff --git a/plancontext/fs.go b/plancontext/fs.go index dc8ca6a2..3287b7ef 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -31,7 +31,7 @@ func (fs *FS) Result() bkgw.Reference { return fs.result } -func (fs *FS) Value() *compiler.Value { +func (fs *FS) MarshalCUE() *compiler.Value { v := compiler.NewValue() if err := v.FillPath(fsIDPath, fs.id); err != nil { panic(err) diff --git a/plancontext/secret.go b/plancontext/secret.go index 0a08ab67..16aefb4d 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -33,7 +33,7 @@ func (s *Secret) PlainText() string { return s.plainText } -func (s *Secret) Value() *compiler.Value { +func (s *Secret) MarshalCUE() *compiler.Value { v := compiler.NewValue() if err := v.FillPath(secretIDPath, s.id); err != nil { panic(err) diff --git a/plancontext/service.go b/plancontext/service.go index 0b2a7bee..6fc6d9b7 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -39,7 +39,7 @@ func (s *Service) NPipe() string { return s.npipe } -func (s *Service) Value() *compiler.Value { +func (s *Service) MarshalCUE() *compiler.Value { v := compiler.NewValue() if err := v.FillPath(serviceIDPath, s.id); err != nil { panic(err) diff --git a/state/input.go b/state/input.go index 22b1fefd..726ec5b7 100644 --- a/state/input.go +++ b/state/input.go @@ -193,7 +193,7 @@ type secretInput string func (i secretInput) Compile(st *State) (*compiler.Value, error) { secret := st.Context.Secrets.New(i.PlainText()) - return secret.Value(), nil + return secret.MarshalCUE(), nil } func (i secretInput) PlainText() string { @@ -296,5 +296,5 @@ type socketInput struct { func (i socketInput) Compile(st *State) (*compiler.Value, error) { service := st.Context.Services.New(i.Unix, i.Npipe) - return service.Value(), nil + return service.MarshalCUE(), nil }