Merge pull request #2033 from jlongtine/action-input-flags

`dagger do` action options flags

cmd/dagger/cmd/do.go

@@ -11,6 +11,7 @@ import (
 
 	"cuelang.org/go/cue"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"go.dagger.io/dagger/cmd/dagger/cmd/common"
 	"go.dagger.io/dagger/cmd/dagger/logger"
@@ -21,8 +22,8 @@ import (
 )
 
 var doCmd = &cobra.Command{
-	Use:   "do [OPTIONS] ACTION [SUBACTION...]",
+	Use: "do ACTION [SUBACTION...]",
-	Short: "Execute a dagger action.",
+	// Short: "Execute a dagger action.",
 	PreRun: func(cmd *cobra.Command, args []string) {
 		// Fix Viper bug for duplicate flags:
 		// https://github.com/spf13/viper/issues/233
@@ -30,10 +31,16 @@ var doCmd = &cobra.Command{
 			panic(err)
 		}
 	},
+	// Don't fail on unknown flags for the first parse
+	FParseErrWhitelist: cobra.FParseErrWhitelist{
+		UnknownFlags: true,
+	},
+	// We're going to take care of flag parsing ourselves
+	DisableFlagParsing: true,
 	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) < 1 {
+		cmd.Flags().Parse(args)
-			doHelpCmd(cmd, nil)
+		if err := viper.BindPFlags(cmd.Flags()); err != nil {
-			return
+			panic(err)
 		}
 
 		var (
@@ -47,6 +54,58 @@ var doCmd = &cobra.Command{
 			lg.Fatal().Err(err).Msg("--platform requires --experimental flag")
 		}
 
+		targetPath := getTargetPath(cmd.Flags().Args())
+
+		daggerPlan, err := loadPlan(viper.GetString("plan"))
+		if err != nil {
+			if viper.GetBool("help") {
+				doHelpCmd(cmd, nil, nil, nil, targetPath, nil)
+				os.Exit(0)
+			}
+			err = fmt.Errorf("failed to load plan: %w", err)
+			doHelpCmd(cmd, nil, nil, nil, targetPath, []string{err.Error()})
+			os.Exit(1)
+		}
+
+		action := daggerPlan.Action().FindByPath(targetPath)
+
+		if action == nil {
+			selectorStrs := []string{}
+			for _, selector := range targetPath.Selectors()[1:] {
+				selectorStrs = append(selectorStrs, selector.String())
+			}
+			targetStr := strings.Join(selectorStrs, " ")
+
+			err = fmt.Errorf("action not found: %s", targetStr)
+			// Find closest action
+			action = daggerPlan.Action().FindClosest(targetPath)
+			targetPath = action.Path
+			doHelpCmd(cmd, daggerPlan, action, nil, action.Path, []string{err.Error()})
+			os.Exit(1)
+		}
+
+		actionFlags := getActionFlags(action)
+
+		cmd.Flags().AddFlagSet(actionFlags)
+
+		cmd.Flags().ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{
+			UnknownFlags: false,
+		}
+		err = cmd.Flags().Parse(args)
+		if err != nil {
+			doHelpCmd(cmd, daggerPlan, action, actionFlags, targetPath, []string{err.Error()})
+			os.Exit(1)
+		}
+
+		if err := viper.BindPFlags(cmd.Flags()); err != nil {
+			panic(err)
+		}
+
+		if len(cmd.Flags().Args()) < 1 || viper.GetBool("help") {
+			doHelpCmd(cmd, daggerPlan, action, actionFlags, targetPath, []string{})
+			os.Exit(0)
+		}
+
 		if f := viper.GetString("log-format"); f == "tty" || f == "auto" && term.IsTerminal(int(os.Stdout.Fd())) {
 			tty, err = logger.NewTTYOutput(os.Stderr)
 			if err != nil {
@@ -61,24 +120,28 @@ var doCmd = &cobra.Command{
 		ctx := lg.WithContext(cmd.Context())
 		cl := common.NewClient(ctx)
 
-		p, err := loadPlan()
+		actionFlags.VisitAll(func(flag *pflag.Flag) {
-		if err != nil {
+			if cmd.Flags().Changed(flag.Name) {
-			lg.Fatal().Err(err).Msg("failed to load plan")
+				fmt.Printf("Changed: %s: %s\n", flag.Name, cmd.Flags().Lookup(flag.Name).Value.String())
-		}
+				flagPath := []cue.Selector{}
-		target := getTargetPath(args)
+				flagPath = append(flagPath, targetPath.Selectors()...)
+				flagPath = append(flagPath, cue.Str(flag.Name))
+				daggerPlan.Source().FillPath(cue.MakePath(flagPath...), viper.Get(flag.Name))
+			}
+		})
 
 		doneCh := common.TrackCommand(ctx, cmd, &telemetry.Property{
 			Name:  "action",
-			Value: target.String(),
+			Value: targetPath.String(),
 		})
 
-		err = cl.Do(ctx, p.Context(), func(ctx context.Context, s *solver.Solver) error {
+		err = cl.Do(ctx, daggerPlan.Context(), func(ctx context.Context, s *solver.Solver) error {
-			return p.Do(ctx, target, s)
+			return daggerPlan.Do(ctx, targetPath, s)
 		})
 
 		<-doneCh
 
-		p.Context().TempDirs.Clean()
+		daggerPlan.Context().TempDirs.Clean()
 
 		if err != nil {
 			lg.Fatal().Err(err).Msg("failed to execute plan")
@@ -86,9 +149,7 @@ var doCmd = &cobra.Command{
 	},
 }
 
-func loadPlan() (*plan.Plan, error) {
+func loadPlan(planPath string) (*plan.Plan, error) {
-	planPath := viper.GetString("plan")
-
 	// support only local filesystem paths
 	// even though CUE supports loading module and package names
 	absPlanPath, err := filepath.Abs(planPath)
@@ -115,27 +176,85 @@ func getTargetPath(args []string) cue.Path {
 	return cue.MakePath(selectors...)
 }
 
-func doHelpCmd(cmd *cobra.Command, _ []string) {
+func doHelpCmd(cmd *cobra.Command, daggerPlan *plan.Plan, action *plan.Action, actionFlags *pflag.FlagSet, target cue.Path, preamble []string) {
 	lg := logger.New()
 
-	fmt.Println(cmd.Short)
+	if len(preamble) > 0 {
+		fmt.Println(strings.Join(preamble, "\n"))
+		fmt.Println()
+	}
 
-	err := printActions(os.Stdout, getTargetPath(cmd.Flags().Args()))
+	target = cue.MakePath(target.Selectors()[1:]...)
+
+	if action != nil {
+		selectorStrs := []string{}
+		for _, selector := range target.Selectors() {
+			selectorStrs = append(selectorStrs, selector.String())
+		}
+		targetStr := strings.Join(selectorStrs, " ")
+		fmt.Printf("Usage: \n  dagger do %s [flags]\n\n", targetStr)
+		if actionFlags != nil {
+			fmt.Println("Options")
+			actionFlags.VisitAll(func(flag *pflag.Flag) {
+				flag.Hidden = false
+			})
+			fmt.Println(actionFlags.FlagUsages())
+			actionFlags.VisitAll(func(flag *pflag.Flag) {
+				flag.Hidden = true
+			})
+		}
+	} else {
+		fmt.Println("Usage: \n  dagger do [flags]")
+	}
+
+	var err error
+	if daggerPlan != nil {
+		err = printActions(daggerPlan, action, os.Stdout, target)
+	}
 
 	fmt.Printf("\n%s", cmd.UsageString())
 
-	if err != nil {
+	if err == nil {
-		lg.Fatal().Err(err).Msg("failed to load plan")
+		lg.Fatal().Err(err)
 	}
 }
 
-func printActions(w io.Writer, target cue.Path) error {
+func getActionFlags(action *plan.Action) *pflag.FlagSet {
-	p, err := loadPlan()
+	flags := pflag.NewFlagSet("action inputs", pflag.ContinueOnError)
-	if err != nil {
+	flags.Usage = func() {}
-		return err
+
+	if action == nil {
+		panic("action is nil")
+	}
+
+	if action.Inputs() == nil {
+		panic("action inputs is nil")
+	}
+
+	for _, input := range action.Inputs() {
+		switch input.Type {
+		case "string":
+			flags.String(input.Name, "", input.Documentation)
+		case "int":
+			flags.Int(input.Name, 0, input.Documentation)
+		case "bool":
+			flags.Bool(input.Name, false, input.Documentation)
+		case "float":
+			flags.Float64(input.Name, 0, input.Documentation)
+		case "number":
+			flags.Float64(input.Name, 0, input.Documentation)
+		default:
+		}
+		flags.MarkHidden(input.Name)
+	}
+	return flags
+}
+
+func printActions(p *plan.Plan, action *plan.Action, w io.Writer, target cue.Path) error {
+	if p == nil {
+		return nil
 	}
 
-	action := p.Action().FindByPath(target)
 	if action == nil {
 		return fmt.Errorf("action %s not found", target.String())
 	}
@@ -168,9 +287,20 @@ func init() {
 	doCmd.Flags().StringArray("cache-from", []string{},
 		"External cache sources (eg. user/app:cache, type=local,src=path/to/dir)")
 
-	doCmd.SetHelpFunc(doHelpCmd)
+	doCmd.SetUsageTemplate(`{{if .HasAvailableSubCommands}}
-
+  {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}
-	if err := viper.BindPFlags(doCmd.Flags()); err != nil {
+Aliases:
-		panic(err)
+  {{.NameAndAliases}}{{end}}{{if .HasExample}}
-	}
+Examples:
+{{.Example}}{{end}}{{if .HasAvailableSubCommands}}
+Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
+  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}
+Flags:
+{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
+Global Flags:
+{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}
+Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}
+  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
+Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
+`)
 }

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/sergi/go-diff v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.11.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tonistiigi/fsutil v0.0.0-20220315205639-9ed612626da3

pkg/universe.dagger.io/aws/cli/test/test.bats

@@ -5,5 +5,5 @@ setup() {
 }
 
 @test "aws/cli" {
-    dagger "do" -p ./sts_get_caller_identity.cue verify
+    dagger "do" -p ./sts_get_caller_identity.cue getCallerIdentity
 }

pkg/universe.dagger.io/aws/test/test.bats

@@ -6,6 +6,6 @@ setup() {
 
 @test "aws" {
     dagger "do" -p ./default_version.cue getVersion
-    dagger "do" -p ./credentials.cue verify
+    dagger "do" -p ./credentials.cue getCallerIdentity
-    dagger "do" -p ./config_file.cue verify
+    dagger "do" -p ./config_file.cue getCallerIdentity
 }

plan/action.go

@@ -2,6 +2,7 @@ package plan
 
 import (
 	"cuelang.org/go/cue"
+	"go.dagger.io/dagger/compiler"
 )
 
 type Action struct {
@@ -10,6 +11,14 @@ type Action struct {
 	Path          cue.Path
 	Documentation string
 	Children      []*Action
+	Value         *compiler.Value
+	inputs        []Input
+}
+
+type Input struct {
+	Name          string
+	Type          string
+	Documentation string
 }
 
 func (a *Action) AddChild(c *Action) {
@@ -31,3 +40,63 @@ func (a *Action) FindByPath(path cue.Path) *Action {
 	}
 	return nil
 }
+
+func (a *Action) FindClosest(path cue.Path) *Action {
+	if a.Path.String() == path.String() {
+		return a
+	}
+
+	commonSubPath := commonSubPath(a.Path, path)
+	if commonSubPath.String() == a.Path.String() {
+		if len(a.Children) > 0 {
+			for _, c := range a.Children {
+				if c.Path.String() == path.String() {
+					return c.FindClosest(path)
+				}
+			}
+		}
+		return a
+	}
+	return nil
+}
+
+func commonSubPath(a, b cue.Path) cue.Path {
+	if a.String() == b.String() {
+		return a
+	}
+	aSelectors := a.Selectors()
+	bSelectors := b.Selectors()
+	commonSelectors := []cue.Selector{}
+	for i := 0; i < len(aSelectors) && i < len(bSelectors); i++ {
+		if aSelectors[i].String() != bSelectors[i].String() {
+			break
+		}
+		commonSelectors = append(commonSelectors, aSelectors[i])
+	}
+	return cue.MakePath(commonSelectors...)
+}
+
+func (a *Action) Inputs() []Input {
+	if a.inputs == nil {
+		cueVal := a.Value.Cue()
+		inputs := []Input{}
+		for iter, _ := cueVal.Fields(cue.All()); iter.Next(); {
+			val := iter.Value()
+			cVal := compiler.Wrap(val)
+
+			_, refPath := val.ReferencePath()
+
+			ik := val.IncompleteKind()
+			validKind := ik == cue.StringKind || ik == cue.NumberKind || ik == cue.BoolKind || ik == cue.IntKind || ik == cue.FloatKind
+			if validKind && !val.IsConcrete() && len(refPath.Selectors()) == 0 {
+				inputs = append(inputs, Input{
+					Name:          iter.Label(),
+					Type:          ik.String(),
+					Documentation: cVal.DocSummary(),
+				})
+			}
+		}
+		a.inputs = inputs
+	}
+	return a.inputs
+}

plan/action_test.go

@@ -0,0 +1,23 @@
+package plan
+
+import (
+	"testing"
+
+	"cuelang.org/go/cue"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClosestSubPath(t *testing.T) {
+	rootPath := cue.MakePath(ActionSelector, cue.Str("test"))
+	path1 := cue.MakePath(ActionSelector, cue.Str("test"), cue.Str("one"))
+	path2 := cue.MakePath(ActionSelector, cue.Str("test"), cue.Str("two"))
+
+	require.Equal(t, "actions.test.one", path1.String())
+	require.Equal(t, "actions.test.two", path2.String())
+	require.Equal(t, "actions.test", commonSubPath(rootPath, path1).String())
+	require.Equal(t, "actions.test", commonSubPath(path1, path2).String())
+
+	path3 := cue.MakePath(ActionSelector, cue.Str("test"), cue.Str("golang"), cue.Str("three"))
+	path4 := cue.MakePath(ActionSelector, cue.Str("test"), cue.Str("java"), cue.Str("three"))
+	require.Equal(t, "actions.test", commonSubPath(path3, path4).String())
+}

plan/plan.go

@@ -191,18 +191,20 @@ func (p *Plan) fillAction() {
 		noOpRunner,
 	)
 
-	p.action = &Action{
+	actionsPath := cue.MakePath(ActionSelector)
-		Name:     ActionSelector.String(),
+	actions := p.source.LookupPath(actionsPath)
-		Hidden:   false,
-		Path:     cue.MakePath(ActionSelector),
-		Children: []*Action{},
-	}
-
-	actions := p.source.LookupPath(cue.MakePath(ActionSelector))
 	if !actions.Exists() {
 		return
 	}
-	p.action.Documentation = actions.DocSummary()
+
+	p.action = &Action{
+		Name:          ActionSelector.String(),
+		Documentation: actions.DocSummary(),
+		Hidden:        false,
+		Path:          actionsPath,
+		Children:      []*Action{},
+		Value:         p.Source().LookupPath(actionsPath),
+	}
 
 	tasks := flow.Tasks()
 
@@ -221,6 +223,7 @@ func (p *Plan) fillAction() {
 					Path:          path,
 					Documentation: v.DocSummary(),
 					Children:      []*Action{},
+					Value:         v,
 				}
 				prevAction.AddChild(a)
 			}

tests/plan.bats

@@ -52,6 +52,14 @@ setup() {
   assert_output --partial ": running \`dagger project update\` may resolve this"
 }
 
+@test "plan/do: check flags" {
+  run "$DAGGER" "do" -p ./plan/do/do_flags.cue test --help
+  assert_output --partial "--doit"
+  assert_output --partial "--message string"
+  assert_output --partial "--name string"
+  assert_output --partial "--num float"
+}
+
 @test "plan/hello" {
   # Europa loader handles the cwd differently, therefore we need to CD into the tree at or below the parent of cue.mod
   cd "$TESTDIR"
@@ -264,7 +272,7 @@ setup() {
 
    # ip address is in a reserved range that should be unroutable
    export BUILDKIT_HOST=tcp://192.0.2.1:1234
-   run timeout 30 "$DAGGER" "do" -p ./plan/do/actions.cue test
+   run timeout 30 "$DAGGER" "do" -p ./plan/do/actions.cue frontend test
    assert_failure
    assert_output --partial "Unavailable: connection error"
 }

tests/plan/do/do_flags.cue

@@ -0,0 +1,61 @@
+package flags
+
+import (
+	"dagger.io/dagger"
+
+	"universe.dagger.io/alpine"
+	"universe.dagger.io/bash"
+)
+
+dagger.#Plan & {
+	client: filesystem: "./test_do": write: contents: actions.test.one.export.files["/output.txt"]
+
+	actions: {
+		foo: "bar"
+		// Pull alpine image
+		image: alpine.#Build & {
+			packages: bash: {}
+		}
+
+		// Run test
+		test: {
+			// Which name?
+			name: string | *"World"
+			// What message?
+			message: string
+			// How many?
+			num: float
+			// on or off?
+			doit: bool | *true
+			// this is foo2
+			foo2: foo
+			// do the first thing
+			one: bash.#Run & {
+				input: image.output
+				script: contents: "echo Hello \(name)! \(doit) > /output.txt"
+				export: files: "/output.txt": string
+			}
+
+			// Do the second thing
+			two: bash.#Run & {
+				input: image.output
+				script: contents: "true"
+			}
+
+			// Do the third thing
+			three: bash.#Run & {
+				input: image.output
+				script: contents: "cat /one/output.txt"
+				mounts: output: {
+					contents: one.export.rootfs
+					dest:     "/one"
+				}
+			}
+		}
+		// !!! DON'T RUN ME !!!
+		notMe: bash.#Run & {
+			input: image.output
+			script: contents: "false"
+		}
+	}
+}

tests/tasks.bats

@@ -11,7 +11,7 @@ setup() {
 }
 
 @test "task: #Push" {
-    "$DAGGER" "do" -p ./tasks/push/push.cue pullContent
+    "$DAGGER" "do" -p ./tasks/push/push.cue pullOutputFile
 }
 
 @test "task: #ReadFile" {