Merge pull request #489 from samalba/inputs2

input list

cmd/dagger/cmd/input/list.go

@@ -9,8 +9,8 @@ import (
 	"dagger.io/go/cmd/dagger/cmd/common"
 	"dagger.io/go/cmd/dagger/logger"
 	"dagger.io/go/dagger"
+	"dagger.io/go/dagger/compiler"
 
-	"cuelang.org/go/cue/ast"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -37,17 +37,6 @@ var listCmd = &cobra.Command{
 
 		environment := common.GetCurrentEnvironmentState(ctx, store)
 
-		// print any persisted inputs
-		if len(environment.Inputs) > 0 {
-			fmt.Println("Saved Inputs:")
-			for _, input := range environment.Inputs {
-				// Todo, how to pull apart an input to print relevant information
-				fmt.Printf("%s: %v\n", input.Key, input.Value)
-			}
-			// add some space
-			fmt.Println()
-		}
-
 		lg = lg.With().
 			Str("environmentName", environment.Name).
 			Str("environmentId", environment.ID).
@@ -59,40 +48,38 @@ var listCmd = &cobra.Command{
 		}
 
 		_, err = c.Do(ctx, environment, func(lCtx context.Context, lDeploy *dagger.Environment, lSolver dagger.Solver) error {
-			inputs, err := lDeploy.ScanInputs()
+			inputs := lDeploy.ScanInputs(ctx)
-			if err != nil {
-				return err
-			}
 
-			fmt.Println("Plan Inputs:")
 			w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
-			fmt.Fprintln(w, "Path\tType")
+			fmt.Fprintln(w, "Input\tType\tValue\tSet by user")
 
-			for _, val := range inputs {
+			for _, inp := range inputs {
-				// check for references
+				isConcrete := (inp.IsConcreteR() == nil)
-				// this is here because it has issues
+				_, hasDefault := inp.Default()
-				// so we wrap it in a flag to control its usage while debugging
+				valStr := "-"
-				_, vals := val.Expr()
+				if isConcrete {
-				if !viper.GetBool("keep-references") {
+					valStr, _ = inp.Cue().String()
-					foundRef := false
-					for _, ve := range vals {
-						s := ve.Source()
-						switch s.(type) {
-						case *ast.Ident:
-							foundRef = true
 				}
+				if hasDefault {
+					valStr = fmt.Sprintf("%s (default)", valStr)
 				}
-					if foundRef {
+
+				if !viper.GetBool("all") {
+					// skip input that is not overridable
+					if !hasDefault && isConcrete {
 						continue
 					}
 				}
 
-				fmt.Fprintf(w, "%s\t%v\n", val.Path(), val)
+				fmt.Fprintf(w, "%s\t%s\t%s\t%t\n",
-
+					inp.Path(),
+					getType(inp),
+					valStr,
+					isUserSet(environment, inp),
+				)
 			}
-			// ensure we flush the output buf
-			w.Flush()
 
+			w.Flush()
 			return nil
 		})
 
@@ -103,8 +90,28 @@ var listCmd = &cobra.Command{
 	},
 }
 
+func isUserSet(env *dagger.EnvironmentState, val *compiler.Value) bool {
+	for _, i := range env.Inputs {
+		if val.Path().String() == i.Key {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getType(val *compiler.Value) string {
+	if val.HasAttr("artifact") {
+		return "dagger.#Artifact"
+	}
+	if val.HasAttr("secret") {
+		return "dagger.#Secret"
+	}
+	return val.Cue().IncompleteKind().String()
+}
+
 func init() {
-	listCmd.Flags().BoolP("keep-references", "R", false, "Try to eliminate references")
+	listCmd.Flags().BoolP("all", "a", false, "List all inputs (include non-overridable)")
 
 	if err := viper.BindPFlags(listCmd.Flags()); err != nil {
 		panic(err)

dagger/compiler/value.go

@@ -145,8 +145,10 @@ func (v *Value) List() ([]*Value, error) {
 }
 
 // Recursive concreteness check.
-func (v *Value) IsConcreteR() error {
+func (v *Value) IsConcreteR(opts ...cue.Option) error {
-	return v.val.Validate(cue.Concrete(true))
+	o := []cue.Option{cue.Concrete(true)}
+	o = append(o, opts...)
+	return v.val.Validate(o...)
 }
 
 func (v *Value) Walk(before func(*Value) bool, after func(*Value)) {
@@ -229,3 +231,42 @@ func (v *Value) IsEmptyStruct() bool {
 func (v *Value) Cue() cue.Value {
 	return v.val
 }
+
+// Returns true if value has a dagger attribute (eg. artifact, secret, input)
+func (v *Value) HasAttr(filter ...string) bool {
+	attrs := v.val.Attributes(cue.ValueAttr)
+
+	for _, attr := range attrs {
+		name := attr.Name()
+		// match `@dagger(...)`
+		if name == "dagger" {
+			// did not provide filter, match any @dagger attr
+			if len(filter) == 0 {
+				return true
+			}
+
+			// loop over args (CSV content in attribute)
+			for i := 0; i < attr.NumArgs(); i++ {
+				key, _ := attr.Arg(i)
+				// one or several values where provided, filter
+				for _, val := range filter {
+					if key == val {
+						return true
+					}
+				}
+			}
+		}
+	}
+
+	return false
+}
+
+func (v *Value) Dereference() *Value {
+	dVal := cue.Dereference(v.val)
+	return v.cc.Wrap(dVal)
+}
+
+func (v *Value) Default() (*Value, bool) {
+	val, hasDef := v.val.Default()
+	return v.cc.Wrap(val), hasDef
+}

dagger/environment.go

@@ -10,7 +10,6 @@ import (
 	"cuelang.org/go/cue"
 	cueflow "cuelang.org/go/tools/flow"
 	"dagger.io/go/dagger/compiler"
-	"dagger.io/go/pkg/cuetils"
 	"dagger.io/go/stdlib"
 
 	"github.com/opentracing/opentracing-go"
@@ -297,11 +296,6 @@ func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc {
 	})
 }
 
-func (e *Environment) ScanInputs() ([]cue.Value, error) {
+func (e *Environment) ScanInputs(ctx context.Context) []*compiler.Value {
-	vals, err := cuetils.ScanForInputs(e.plan.Cue())
+	return ScanInputs(ctx, e.plan)
-	if err != nil {
-		return nil, err
-	}
-
-	return vals, nil
 }

dagger/inputs_scan.go

@@ -0,0 +1,68 @@
+package dagger
+
+import (
+	"context"
+
+	"cuelang.org/go/cue"
+	"dagger.io/go/dagger/compiler"
+	"github.com/rs/zerolog/log"
+)
+
+func isReference(val cue.Value) bool {
+	isRef := func(v cue.Value) bool {
+		_, ref := v.ReferencePath()
+
+		if ref.String() == "" || v.Path().String() == ref.String() {
+			// not a reference
+			return false
+		}
+
+		for _, s := range ref.Selectors() {
+			if s.IsDefinition() {
+				// if we reference to a definition, we skip the check
+				return false
+			}
+		}
+
+		return true
+	}
+
+	op, vals := val.Expr()
+	if op == cue.NoOp {
+		return isRef(val)
+	}
+
+	for _, v := range vals {
+		// if the expr has an op (& or |, etc...), check the expr values, recursively
+		if isReference(v) {
+			return true
+		}
+	}
+
+	return isRef(val)
+}
+
+func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
+	lg := log.Ctx(ctx)
+	inputs := []*compiler.Value{}
+
+	value.Walk(
+		func(val *compiler.Value) bool {
+			if isReference(val.Cue()) {
+				lg.Debug().Str("value.Path", val.Path().String()).Msg("found reference, stop walk")
+				return false
+			}
+
+			if !val.HasAttr("input") {
+				return true
+			}
+
+			lg.Debug().Str("value.Path", val.Path().String()).Msg("found input")
+			inputs = append(inputs, val)
+
+			return true
+		}, nil,
+	)
+
+	return inputs
+}

examples/simple-s3/main.cue

@@ -8,13 +8,13 @@ import (
 
 // AWS Config for credentials and default region
 awsConfig: aws.#Config & {
-	region: *"us-east-1" | string
+	region: *"us-east-1" | string @dagger(input)
 }
 
 // Name of the S3 bucket to use
-bucket: *"dagger-io-examples" | string
+bucket: *"dagger-io-examples" | string @dagger(input)
 
-source: dagger.#Artifact
+source: dagger.#Artifact @dagger(input)
 url:    "\(deploy.url)index.html"
 
 deploy: s3.#Put & {

pkg/cuetils/scan.go

@@ -1,136 +0,0 @@
-package cuetils
-
-import (
-	"cuelang.org/go/cue"
-)
-
-// ScanForInputs walks a Value looking for potential inputs
-// - non-concrete values or values with defaults
-// - exclude @dagger(computed) and #up
-// - exclude values which have references
-func ScanForInputs(value cue.Value) ([]cue.Value, error) {
-	var (
-		vals []cue.Value
-		err  error
-	)
-
-	// walk before function, bool return is if the walk should recurse again
-	before := func(v cue.Value) (bool, error) {
-		// explicit phase
-		// look for #up
-		label, _ := v.Label()
-		if label == "#up" {
-			return false, nil
-		}
-
-		// look to exclude any @dagger(computed)
-		attrs := v.Attributes(cue.ValueAttr)
-		for _, attr := range attrs {
-			name := attr.Name()
-			// match `@dagger(...)`
-			if name == "dagger" {
-				// loop over args (CSV content in attribute)
-				for i := 0; i < attr.NumArgs(); i++ {
-					key, _ := attr.Arg(i)
-
-					// we found an explicit computed value
-					if key == "computed" {
-						return false, nil
-					}
-				}
-			}
-		}
-
-		// inference phase
-		switch v.IncompleteKind() {
-		case cue.StructKind:
-			return true, nil
-
-		case cue.ListKind:
-			if !v.IsConcrete() {
-				vals = append(vals, v)
-				return false, nil
-			}
-			return true, nil
-
-		default:
-
-			// a leaf with default?
-			_, has := v.Default()
-			if has {
-				vals = append(vals, v)
-				// recurse here?
-				return false, nil
-			}
-
-			// is this leaf not concrete? (should cause an error)
-			if v.Validate(cue.Concrete(true), cue.Optional(true)) != nil {
-				vals = append(vals, v)
-			}
-
-			return false, nil
-		}
-	}
-
-	// walk
-	err = walkValue(value, before, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return vals, nil
-}
-
-// walkValue is a custome walk function so that we recurse into more types than CUE's buildin walk
-// specificially, we need to customize the options to val.Fields when val is a struct
-func walkValue(val cue.Value, before func(cue.Value) (bool, error), after func(cue.Value) error) error {
-	if before != nil {
-		recurse, err := before(val)
-		if err != nil {
-			return err
-		}
-
-		// should we recurse into fields
-		if recurse {
-			switch val.IncompleteKind() {
-			case cue.StructKind:
-				// provide custom args to ensure we walk nested defs
-				// and that optionals are included
-				iter, err := val.Fields(
-					cue.Definitions(true),
-					cue.Optional(true),
-				)
-				if err != nil {
-					return err
-				}
-				for iter.Next() {
-					err := walkValue(iter.Value(), before, after)
-					if err != nil {
-						return err
-					}
-				}
-
-			case cue.ListKind:
-				iter, err := val.List()
-				if err != nil {
-					return err
-				}
-				for iter.Next() {
-					err := walkValue(iter.Value(), before, after)
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-	}
-
-	if after != nil {
-		err := after(val)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

stdlib/aws/aws.cue

@@ -9,11 +9,11 @@ import (
 // Base AWS Config
 #Config: {
 	// AWS region
-	region: string
+	region: string @dagger(input)
 	// AWS access key
-	accessKey: dagger.#Secret
+	accessKey: dagger.#Secret @dagger(input)
 	// AWS secret key
-	secretKey: dagger.#Secret
+	secretKey: dagger.#Secret @dagger(input)
 }
 
 // Re-usable aws-cli component

stdlib/aws/s3/s3.cue

@@ -12,22 +12,22 @@ import (
 	config: aws.#Config
 
 	// Source Artifact to upload to S3
-	source?: dagger.#Artifact
+	source?: dagger.#Artifact @dagger(input)
 
 	// Source inlined as a string to upload to S3
-	sourceInline?: string
+	sourceInline?: string @dagger(input)
 
 	// Target S3 URL (eg. s3://<bucket-name>/<path>/<sub-path>)
-	target: string
+	target: string @dagger(input)
 
 	// Object content type
-	contentType: string | *""
+	contentType: string | *"" @dagger(input)
 
 	// URL of the uploaded S3 object
-	url: out
+	url: out @dagger(output)
 
 	// Always write the object to S3
-	always?: bool
+	always?: bool @dagger(input)
 
 	out: string
 	aws.#Script & {

stdlib/dagger/dagger.cue

@@ -8,6 +8,7 @@ import (
 // May be passed as user input, or computed by a buildkit pipeline
 
 #Artifact: {
+	@dagger(artifact)
 	#up: [...op.#Op]
 	_
 	...
@@ -17,5 +18,6 @@ import (
 // FIXME: currently aliased as a string to mark secrets
 // this requires proper support.
 #Secret: {
+	@dagger(secret)
 	string | bytes
 }