cue native: environments can reference a module instead of embedding

one.

Fixes #631

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

cmd/dagger/cmd/compute.go

@@ -38,7 +38,9 @@ var computeCmd = &cobra.Command{
 		st := &state.State{
 			Name: "FIXME",
 			Path: args[0],
-			Plan: args[0],
+			Plan: state.Plan{
+				Module: args[0],
+			},
 		}
 
 		for _, input := range viper.GetStringSlice("input-string") {

cmd/dagger/cmd/edit.go

@@ -65,6 +65,7 @@ var editCmd = &cobra.Command{
 			lg.Fatal().Err(err).Msg("failed to decode file")
 		}
 		st.Name = newState.Name
+		st.Plan = newState.Plan
 		st.Inputs = newState.Inputs
 		if err := workspace.Save(ctx, st); err != nil {
 			lg.Fatal().Err(err).Msg("failed to save state")

cmd/dagger/cmd/new.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -32,17 +34,40 @@ var newCmd = &cobra.Command{
 				Msg("cannot use option -e,--environment for this command")
 		}
 		name := args[0]
-		ws, err := workspace.Create(ctx, name)
+
+		module := viper.GetString("module")
+		if module != "" {
+			p, err := filepath.Abs(module)
+			if err != nil {
+				lg.Fatal().Err(err).Str("path", module).Msg("unable to resolve path")
+			}
+
+			if !strings.HasPrefix(p, workspace.Path) {
+				lg.Fatal().Err(err).Str("path", module).Msg("module is outside the workspace")
+			}
+			p, err = filepath.Rel(workspace.Path, p)
+			if err != nil {
+				lg.Fatal().Err(err).Str("path", module).Msg("unable to resolve path")
+			}
+			if !strings.HasPrefix(p, ".") {
+				p = "./" + p
+			}
+			module = p
+		}
+
+		ws, err := workspace.Create(ctx, name, module, viper.GetString("package"))
 		if err != nil {
 			lg.Fatal().Err(err).Msg("failed to create environment")
 		}
 
 		lg.Info().Str("name", name).Msg("created new empty environment")
-		lg.Info().Str("name", name).Msg(fmt.Sprintf("to add code to the plan, copy or create cue files under: %s", ws.Plan))
+		lg.Info().Str("name", name).Msg(fmt.Sprintf("to add code to the plan, copy or create cue files under: %s", ws.Plan.Module))
 	},
 }
 
 func init() {
+	newCmd.Flags().StringP("module", "m", "", "references the local path of the cue module to use as a plan, relative to the workspace root")
+	newCmd.Flags().StringP("package", "p", "", "references the name of the Cue package within the module to use as a plan. Default: defer to cue loader")
 	if err := viper.BindPFlags(newCmd.Flags()); err != nil {
 		panic(err)
 	}

environment/environment.go

@@ -65,10 +65,6 @@ func (e *Environment) Name() string {
 	return e.state.Name
 }
 
-func (e *Environment) PlanSource() state.Input {
-	return e.state.PlanSource()
-}
-
 func (e *Environment) Plan() *compiler.Value {
 	return e.plan
 }
@@ -86,7 +82,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan")
 	defer span.Finish()
 
-	planSource, err := e.state.PlanSource().Compile("", e.state)
+	planSource, err := e.state.Plan.Source().Compile("", e.state)
 	if err != nil {
 		return err
 	}
@@ -102,7 +98,11 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
 		stdlib.Path: stdlib.FS,
 		"/":         p.FS(),
 	}
-	plan, err := compiler.Build(sources)
+	args := []string{}
+	if pkg := e.state.Plan.Package; pkg != "" {
+		args = append(args, pkg)
+	}
+	plan, err := compiler.Build(sources, args...)
 	if err != nil {
 		return fmt.Errorf("plan config: %w", compiler.Err(err))
 	}
@@ -157,7 +157,7 @@ func (e *Environment) LocalDirs() map[string]string {
 	}
 
 	// 2. Scan the plan
-	plan, err := e.state.PlanSource().Compile("", e.state)
+	plan, err := e.state.Plan.Source().Compile("", e.state)
 	if err != nil {
 		panic(err)
 	}

state/state.go

@@ -8,8 +8,8 @@ type State struct {
 	// Workspace path
 	Workspace string `yaml:"-"`
 
-	// Plan path
-	Plan string `yaml:"-"`
+	// Plan
+	Plan Plan `yaml:"plan"`
 
 	// Human-friendly environment name.
 	// A environment may have more than one name.
@@ -23,10 +23,15 @@ type State struct {
 	Computed string `yaml:"-"`
 }
 
+type Plan struct {
+	Module  string `yaml:"module,omitempty"`
+	Package string `yaml:"package,omitempty"`
+}
+
 // Cue module containing the environment plan
 // The input's top-level artifact is used as a module directory.
-func (s *State) PlanSource() Input {
-	return DirInput(s.Plan, []string{"*.cue", "cue.mod"}, []string{})
+func (p *Plan) Source() Input {
+	return DirInput(p.Module, []string{}, []string{})
 }
 
 func (s *State) SetInput(key string, value Input) error {

state/workspace.go

@@ -116,11 +116,14 @@ func (w *Workspace) List(ctx context.Context) ([]*State, error) {
 		}
 		st, err := w.Get(ctx, f.Name())
 		if err != nil {
+			// If the environment doesn't exist (e.g. no values.yaml, skip silently)
+			if !errors.Is(err, ErrNotExist) {
 				log.
 					Ctx(ctx).
 					Err(err).
 					Str("name", f.Name()).
 					Msg("failed to load environment")
+			}
 			continue
 		}
 		environments = append(environments, st)
@@ -143,6 +146,9 @@ func (w *Workspace) Get(ctx context.Context, name string) (*State, error) {
 
 	manifest, err := os.ReadFile(path.Join(envPath, manifestFile))
 	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, ErrNotExist
+		}
 		return nil, err
 	}
 	manifest, err = keychain.Decrypt(ctx, manifest)
@@ -155,7 +161,19 @@ func (w *Workspace) Get(ctx context.Context, name string) (*State, error) {
 		return nil, err
 	}
 	st.Path = envPath
-	st.Plan = path.Join(envPath, planDir)
+	// Backward compat: if no plan module has been provided,
+	// use `.dagger/env/<name>/plan`
+	if st.Plan.Module == "" {
+		planPath := path.Join(envPath, planDir)
+		if _, err := os.Stat(planPath); err != nil {
+			return nil, fmt.Errorf("missing plan information for %q", name)
+		}
+		planRelPath, err := filepath.Rel(w.Path, planPath)
+		if err != nil {
+			return nil, err
+		}
+		st.Plan.Module = planRelPath
+	}
 	st.Workspace = w.Path
 
 	computed, err := os.ReadFile(path.Join(envPath, stateDir, computedFile))
@@ -211,7 +229,7 @@ func (w *Workspace) Save(ctx context.Context, st *State) error {
 	return nil
 }
 
-func (w *Workspace) Create(ctx context.Context, name string) (*State, error) {
+func (w *Workspace) Create(ctx context.Context, name, module, pkg string) (*State, error) {
 	envPath, err := filepath.Abs(w.envPath(name))
 	if err != nil {
 		return nil, err
@@ -225,22 +243,33 @@ func (w *Workspace) Create(ctx context.Context, name string) (*State, error) {
 		return nil, err
 	}
 
-	// Plan directory
-	if err := os.Mkdir(path.Join(envPath, planDir), 0755); err != nil {
-		if errors.Is(err, os.ErrExist) {
-			return nil, ErrExist
-		}
+	manifestPath := path.Join(envPath, manifestFile)
+
+	// Backward compat: if no plan module has been provided,
+	// use `.dagger/env/<name>/plan`
+	if module == "" {
+		planPath := path.Join(envPath, planDir)
+		if err := os.Mkdir(planPath, 0755); err != nil {
 			return nil, err
 		}
 
-	manifestPath := path.Join(envPath, manifestFile)
+		planRelPath, err := filepath.Rel(w.Path, planPath)
+		if err != nil {
+			return nil, err
+		}
+		module = planRelPath
+	}
 
 	st := &State{
 		Path:      envPath,
 		Workspace: w.Path,
-		Plan:      path.Join(envPath, planDir),
+		Plan: Plan{
+			Module:  module,
+			Package: pkg,
+		},
 		Name: name,
 	}
+
 	data, err := yaml.Marshal(st)
 	if err != nil {
 		return nil, err

tests/cli.bats

@@ -38,6 +38,52 @@ setup() {
     assert_failure
 }
 
+# create different environments from the same module
+@test "dagger new: modules" {
+    "$DAGGER" init
+
+    ln -s "$TESTDIR"/cli/input/simple "$DAGGER_WORKSPACE"/plan
+
+    "$DAGGER" new "a" --module "$DAGGER_WORKSPACE"/plan
+    "$DAGGER" new "b" --module "$DAGGER_WORKSPACE"/plan
+
+    "$DAGGER" input -e "a" text "input" "a"
+    "$DAGGER" input -e "b" text "input" "b"
+
+    "$DAGGER" up -e "a"
+    "$DAGGER" up -e "b"
+
+    run "$DAGGER" query -l error -e "a" input -f text
+    assert_success
+    assert_output "a"
+
+    run "$DAGGER" query -l error -e "b" input -f text
+    assert_success
+    assert_output "b"
+}
+
+# create different environments from the same module,
+# using different packages.
+@test "dagger new: packages" {
+    "$DAGGER" init
+
+    ln -s "$TESTDIR"/cli/packages "$DAGGER_WORKSPACE"/plan
+
+    "$DAGGER" new "a" --module "$DAGGER_WORKSPACE"/plan --package dagger.io/test/a
+    "$DAGGER" new "b" --module "$DAGGER_WORKSPACE"/plan --package dagger.io/test/b
+
+    "$DAGGER" up -e "a"
+    "$DAGGER" up -e "b"
+
+    run "$DAGGER" query -l error -e "a" exp -f text
+    assert_success
+    assert_output "a"
+
+    run "$DAGGER" query -l error -e "b" exp -f text
+    assert_success
+    assert_output "b"
+}
+
 @test "dagger query" {
     "$DAGGER" init
 

tests/cli/packages/a/main.cue

@@ -0,0 +1,19 @@
+package a
+
+import "dagger.io/dagger/op"
+
+exp: {
+	string
+	#up: [
+		op.#FetchContainer & {ref: "busybox"},
+		op.#Exec & {
+			args: ["sh", "-c", """
+				printf a > /export
+				"""]
+		},
+		op.#Export & {
+			source: "/export"
+			format: "string"
+		},
+	]
+}

tests/cli/packages/b/main.cue

@@ -0,0 +1,19 @@
+package b
+
+import "dagger.io/dagger/op"
+
+exp: {
+	string
+	#up: [
+		op.#FetchContainer & {ref: "busybox"},
+		op.#Exec & {
+			args: ["sh", "-c", """
+				printf b > /export
+				"""]
+		},
+		op.#Export & {
+			source: "/export"
+			format: "string"
+		},
+	]
+}

tests/cli/packages/cue.mod/module.cue

@@ -0,0 +1 @@
+module: "dagger.io/test"

tests/helpers.bash

@@ -20,11 +20,11 @@ common_setup() {
 dagger_new_with_plan() {
     local name="$1"
     local sourcePlan="$2"
-    local targetPlan="$DAGGER_WORKSPACE"/.dagger/env/"$name"/plan
+    local targetPlan="$DAGGER_WORKSPACE"/"$name"
 
-    "$DAGGER" new "$name"
-    rmdir "$targetPlan"
     ln -s "$sourcePlan" "$targetPlan"
+    "$DAGGER" new "$name" --module "$targetPlan"
+
 }
 
 skip_unless_secrets_available() {