universe vendoring

Rather than injecting universe at runtime, this change will vendor
alpha.dagger.io in `cue.mod` directly.

Fixes #700

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

.dagger/env/test-core/.gitignore

@@ -1,2 +0,0 @@
-# dagger state
-state/**

.dagger/env/test-core/values.yaml

@@ -1,27 +0,0 @@
-plan:
-    module: .dagger/env/test-core/plan
-name: test-core
-inputs:
-    dir:
-        dir:
-            path: ./tests
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhOW1rUy9vNFBDaXRDUHJu
-            ZVZYM1FucVorRDdUcmRaYmg3eTlTNzhhYWdjCndWZWxhZnhCZG4xZU9JQ1VyMXdR
-            OHY0TS81bk9FL2JuUEhuTmcxa29ORGcKLS0tIGxJUzNrZmRBNHZGRFY2Z01QK2JP
-            MlM1Ukdqbi9SQ0pqTi9FZ3MxN2E2QmsKHwd7P6KHPVdynOoto1jf3G4+5+vf87wU
-            HX1KD7Od5wRdBwn7r3OS8mdvuNIYpJDUb5YDrfjQypt020ohLocNiA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2021-06-17T14:07:53Z"
-    mac: ENC[AES256_GCM,data:afYut7wdvsJQgxlBg6NEV6DWk8vPi81mZgQAuWb4oe4WJeI1T9cYdtjOHPlmIpjqb86VQHJ29YTzektei2+k+VBawQxcsvefK7X1QboJTfMKLfsiug4qzNWjc7JZDvTb6dsDFM1U96gjSoAIVwdLMnucbu3681Fd7qSQgqNS61Q=,iv:ZQDHzXp0RJcUI4RtOVjdepV8zTa2kIHQhAltLkudDck=,tag:AnSFi1mKrEuXSqE4R+g7dw==,type:str]
-    pgp: []
-    encrypted_suffix: secret
-    version: 3.7.1

environment/environment.go

@@ -12,7 +12,6 @@ import (
 	"go.dagger.io/dagger/compiler"
 	"go.dagger.io/dagger/solver"
 	"go.dagger.io/dagger/state"
-	"go.dagger.io/dagger/stdlib"
 
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
@@ -82,6 +81,18 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan")
 	defer span.Finish()
 
+	// FIXME: universe vendoring
+	// This is already done on `dagger init` and shouldn't be done here too.
+	// However:
+	// 1) As of right now, there's no way to update universe through the
+	// CLI, so we are lazily updating on `dagger up` using the embedded `universe`
+	// 2) For backward compatibility: if the workspace was `dagger
+	// init`-ed before we added support for vendoring universe, it might not
+	// contain a `cue.mod`.
+	if err := e.state.VendorUniverse(ctx); err != nil {
+		return err
+	}
+
 	planSource, err := e.state.Source().Compile("", e.state)
 	if err != nil {
 		return err
@@ -95,8 +106,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
 
 	// Build a Cue config by overlaying the source with the stdlib
 	sources := map[string]fs.FS{
-		stdlib.Path: stdlib.FS,
+		"/": p.FS(),
-		"/":         p.FS(),
 	}
 	args := []string{}
 	if pkg := e.state.Plan.Package; pkg != "" {

state/state.go

@@ -1,5 +1,10 @@
 package state
 
+import (
+	"context"
+	"path"
+)
+
 // Contents of an environment serialized to a file
 type State struct {
 	// State path
@@ -28,11 +33,26 @@ func (s *State) Source() Input {
 	w := s.Workspace
 	// FIXME: backward compatibility
 	if mod := s.Plan.Module; mod != "" {
-		w = mod
+		w = path.Join(w, mod)
 	}
 	return DirInput(w, []string{}, []string{})
 }
 
+// VendorUniverse vendors the latest (built-in) version of the universe into the
+// environment's `cue.mod`.
+// FIXME: This has nothing to do in `State` and should be tied to a `Workspace`.
+// However, since environments could point to different modules before, we have
+// to handle vendoring on a per environment basis.
+func (s *State) VendorUniverse(ctx context.Context) error {
+	w := s.Workspace
+	// FIXME: backward compatibility
+	if mod := s.Plan.Module; mod != "" {
+		w = path.Join(w, mod)
+	}
+
+	return vendorUniverse(ctx, w)
+}
+
 type Plan struct {
 	Module  string `yaml:"module,omitempty"`
 	Package string `yaml:"package,omitempty"`

state/workspace.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"go.dagger.io/dagger/keychain"
+	"go.dagger.io/dagger/stdlib"
 	"gopkg.in/yaml.v3"
 )
 
@@ -51,6 +52,11 @@ func Init(ctx context.Context, dir string) (*Workspace, error) {
 	if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil {
 		return nil, err
 	}
+
+	if err := vendorUniverse(ctx, root); err != nil {
+		return nil, err
+	}
+
 	return &Workspace{
 		Path: root,
 	}, nil
@@ -335,3 +341,63 @@ func (w *Workspace) cleanPackageName(ctx context.Context, pkg string) (string, e
 
 	return p, nil
 }
+
+func cueModInit(ctx context.Context, p string) error {
+	lg := log.Ctx(ctx)
+
+	mod := path.Join(p, "cue.mod")
+	if err := os.Mkdir(mod, 0755); err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+
+	modFile := path.Join(mod, "module.cue")
+	if _, err := os.Stat(modFile); err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+
+		lg.Debug().Str("mod", p).Msg("initializing cue.mod")
+
+		if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
+			return err
+		}
+	}
+
+	if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+	if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func vendorUniverse(ctx context.Context, p string) error {
+	// ensure cue module is initialized
+	if err := cueModInit(ctx, p); err != nil {
+		return err
+	}
+
+	// add universe to `.gitignore`
+	if err := os.WriteFile(
+		path.Join(p, "cue.mod", "pkg", ".gitignore"),
+		[]byte(fmt.Sprintf("# dagger universe\n%s\n", stdlib.PackageName)),
+		0600,
+	); err != nil {
+		return err
+	}
+
+	log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
+	if err := stdlib.Vendor(ctx, p); err != nil {
+		return err
+	}
+
+	return nil
+}

stdlib/stdlib.go

@@ -1,8 +1,13 @@
 package stdlib
 
 import (
+	"context"
 	"embed"
+	"fmt"
+	"io/fs"
+	"os"
 	"path"
+	"path/filepath"
 )
 
 var (
@@ -13,3 +18,38 @@ var (
 	PackageName = "alpha.dagger.io"
 	Path        = path.Join("cue.mod", "pkg", PackageName)
 )
+
+func Vendor(ctx context.Context, mod string) error {
+	// Remove any existing copy of the universe
+	if err := os.RemoveAll(path.Join(mod, Path)); err != nil {
+		return err
+	}
+
+	// Write the current version
+	return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !entry.Type().IsRegular() {
+			return nil
+		}
+
+		if filepath.Ext(entry.Name()) != ".cue" {
+			return nil
+		}
+
+		contents, err := fs.ReadFile(FS, p)
+		if err != nil {
+			return fmt.Errorf("%s: %w", p, err)
+		}
+
+		overlayPath := path.Join(mod, Path, p)
+
+		if err := os.MkdirAll(filepath.Dir(overlayPath), 0755); err != nil {
+			return err
+		}
+
+		return os.WriteFile(overlayPath, contents, 0600)
+	})
+}

tests/core.bats

@@ -15,30 +15,34 @@ setup() {
 # at the beginning of each new-style test.
 
 @test "core: inputs & outputs" {
-   # Use native Dagger environment here
+    dagger init
-   unset DAGGER_WORKSPACE
 
-   # List available inputs
+    dagger_new_with_plan test-core "$TESTDIR"/core/inputs-outputs
-   run dagger -e test-core input list
-   assert_success
-   assert_output --partial 'name'
-   assert_output --partial 'dir'
 
-   # Set text input
+    # List available inputs
-   dagger -e test-core input text name Bob
+    run dagger -e test-core input list
-   run dagger -e test-core up
+    assert_success
-   assert_success
+    assert_output --partial 'name'
-   assert_output --partial 'Hello, Bob!'
+    assert_output --partial 'dir'
 
-   run dagger -e test-core output list
+    # Set dir input
-   assert_success
+    dagger -e test-core input dir dir "$DAGGER_WORKSPACE"
-   assert_output --partial 'message  "Hello, Bob!"'
 
-   # Unset text input
+    # Set text input
-   dagger -e test-core input unset name
+    dagger -e test-core input text name Bob
-   run dagger -e test-core up
+    run dagger -e test-core up
-   assert_success
+    assert_success
-   assert_output --partial 'Hello, world!'
+    assert_output --partial 'Hello, Bob!'
+
+    run dagger -e test-core output list
+    assert_success
+    assert_output --partial 'message  "Hello, Bob!"'
+
+    # Unset text input
+    dagger -e test-core input unset name
+    run dagger -e test-core up
+    assert_success
+    assert_output --partial 'Hello, world!'
 }
 
 

tests/ops/exec/undefined/with_pkg_def/cue.mod/module.cue

@@ -1 +1 @@
-module: "alpha.dagger.io/testing"
+module: "test.dagger.io/testing"

tests/ops/exec/undefined/with_pkg_def/main.cue

@@ -2,7 +2,7 @@ package testing
 
 import (
 	"alpha.dagger.io/dagger/op"
-	"alpha.dagger.io/def"
+	"test.dagger.io/def"
 )
 
 #up: [

tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/module.cue

@@ -1 +1 @@
-module: "alpha.dagger.io/testing"
+module: "test.dagger.io/testing"

tests/ops/exec/undefined/with_pkg_mandatory/main.cue

@@ -1,7 +1,7 @@
 package testing
 
 import (
-	"alpha.dagger.io/nonoptional"
+	"test.dagger.io/nonoptional"
 	"alpha.dagger.io/dagger/op"
 )
 

tests/ops/exec/undefined/with_pkg_optional/cue.mod/module.cue

@@ -1 +1 @@
-module: "alpha.dagger.io/testing"
+module: "test.dagger.io/testing"

tests/ops/exec/undefined/with_pkg_optional/main.cue

@@ -1,7 +1,7 @@
 package testing
 
 import (
-	"alpha.dagger.io/optional"
+	"test.dagger.io/optional"
 	"alpha.dagger.io/dagger/op"
 )