vendoring: support multiple modules

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

pkg/pkg.go

@@ -3,24 +3,112 @@ package pkg
 import (
 	"context"
 	"embed"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
+
+	"github.com/gofrs/flock"
+	"github.com/rs/zerolog/log"
 )
 
 var (
 	// FS contains the filesystem of the stdlib.
-	//go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue alpha.dagger.io/europa/dagger/*.cue alpha.dagger.io/europa/dagger/engine/*.cue
+	//go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue dagger.io/**/*.cue dagger.io/**/*/*.cue
 	FS embed.FS
-
-	AlphaModule   = "alpha.dagger.io"
-	EnginePackage = fmt.Sprintf("%s/europa/dagger/engine", AlphaModule)
 )
 
-func Vendor(ctx context.Context, dest string) error {
+var (
-	// Write the current version
+	AlphaModule   = "alpha.dagger.io"
+	DaggerModule  = "dagger.io"
+	EnginePackage = fmt.Sprintf("%s/dagger/engine", DaggerModule)
+
+	modules = []string{
+		AlphaModule,
+		DaggerModule,
+	}
+
+	lockFilePath = "dagger.lock"
+)
+
+func Vendor(ctx context.Context, p string) error {
+	if p == "" {
+		p = getCueModParent()
+	}
+
+	cuePkgDir := path.Join(p, "cue.mod", "pkg")
+	if err := os.MkdirAll(cuePkgDir, 0755); err != nil {
+		return err
+	}
+
+	// Lock this function so no more than 1 process can run it at once.
+	lockFile := path.Join(cuePkgDir, lockFilePath)
+	l := flock.New(lockFile)
+	if err := l.Lock(); err != nil {
+		return err
+	}
+	defer func() {
+		l.Unlock()
+		os.Remove(lockFile)
+	}()
+
+	// ensure cue module is initialized
+	if err := cueModInit(ctx, p); err != nil {
+		return err
+	}
+
+	// generate `.gitignore`
+	if err := os.WriteFile(
+		path.Join(cuePkgDir, ".gitignore"),
+		[]byte(fmt.Sprintf("# generated by dagger\n%s\n%s\ndagger.lock\n", AlphaModule, DaggerModule)),
+		0600,
+	); err != nil {
+		return err
+	}
+
+	log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring packages")
+
+	// Unpack modules in a temporary directory
+	unpackDir, err := os.MkdirTemp(cuePkgDir, "vendor-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(unpackDir)
+
+	if err := extractModules(unpackDir); err != nil {
+		return err
+	}
+
+	for _, module := range modules {
+		// Semi-atomic swap of the module
+		//
+		// The following basically does:
+		// $ rm -rf cue.mod/pkg/MODULE.old
+		// $ mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old
+		// $ mv VENDOR/MODULE cue.mod/pkg/MODULE
+		// $ rm -rf cue.mod/pkg/MODULE.old
+
+		moduleDir := path.Join(cuePkgDir, module)
+		backupModuleDir := moduleDir + ".old"
+		if err := os.RemoveAll(backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		if err := os.Rename(moduleDir, backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		defer os.RemoveAll(backupModuleDir)
+
+		if err := os.Rename(path.Join(unpackDir, module), moduleDir); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func extractModules(dest string) error {
 	return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -48,3 +136,56 @@ func Vendor(ctx context.Context, dest string) error {
 		return os.WriteFile(overlayPath, contents, 0600)
 	})
 }
+
+// getCueModParent traverses the directory tree up through ancestors looking for a cue.mod folder
+func getCueModParent() string {
+	cwd, _ := os.Getwd()
+	parentDir := cwd
+
+	for {
+		if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) {
+			break // found it!
+		}
+
+		parentDir = filepath.Dir(parentDir)
+
+		if parentDir == string(os.PathSeparator) {
+			// reached the root
+			parentDir = cwd // reset to working directory
+			break
+		}
+	}
+
+	return parentDir
+}
+
+func cueModInit(ctx context.Context, parentDir string) error {
+	lg := log.Ctx(ctx)
+
+	modDir := path.Join(parentDir, "cue.mod")
+	modFile := path.Join(modDir, "module.cue")
+	if _, err := os.Stat(modFile); err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+
+		lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
+
+		if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
+			return err
+		}
+	}
+
+	if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+	if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+
+	return nil
+}

plan/plan.go

@@ -11,10 +11,10 @@ import (
 	"github.com/rs/zerolog/log"
 	"go.dagger.io/dagger/compiler"
 	"go.dagger.io/dagger/environment"
+	"go.dagger.io/dagger/pkg"
 	"go.dagger.io/dagger/plan/task"
 	"go.dagger.io/dagger/plancontext"
 	"go.dagger.io/dagger/solver"
-	"go.dagger.io/dagger/state"
 	"go.opentelemetry.io/otel"
 )
 
@@ -33,8 +33,8 @@ type Config struct {
 func Load(ctx context.Context, cfg Config) (*Plan, error) {
 	log.Ctx(ctx).Debug().Interface("args", cfg.Args).Msg("loading plan")
 
-	// FIXME: universe vendoring
+	// FIXME: vendoring path
-	if err := state.VendorUniverse(ctx, ""); err != nil {
+	if err := pkg.Vendor(ctx, ""); err != nil {
 		return nil, err
 	}
 

state/project.go

@@ -10,7 +10,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/gofrs/flock"
 	"github.com/rs/zerolog/log"
 	"go.dagger.io/dagger/keychain"
 	"go.dagger.io/dagger/pkg"
@@ -32,7 +31,6 @@ const (
 	planDir      = "plan"
 	manifestFile = "values.yaml"
 	computedFile = "computed.json"
-	lockFilePath = "dagger.lock"
 )
 
 type Project struct {
@@ -57,7 +55,7 @@ func Init(ctx context.Context, dir string) (*Project, error) {
 		return nil, err
 	}
 
-	if err := VendorUniverse(ctx, root); err != nil {
+	if err := pkg.Vendor(ctx, root); err != nil {
 		return nil, err
 	}
 
@@ -354,133 +352,3 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err
 
 	return p, nil
 }
-
-func cueModInit(ctx context.Context, parentDir string) error {
-	lg := log.Ctx(ctx)
-
-	modDir := path.Join(parentDir, "cue.mod")
-	modFile := path.Join(modDir, "module.cue")
-	if _, err := os.Stat(modFile); err != nil {
-		if !errors.Is(err, os.ErrNotExist) {
-			return err
-		}
-
-		lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
-
-		if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
-			return err
-		}
-	}
-
-	if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
-		if !errors.Is(err, os.ErrExist) {
-			return err
-		}
-	}
-	if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
-		if !errors.Is(err, os.ErrExist) {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func VendorUniverse(ctx context.Context, p string) error {
-	if p == "" {
-		p = getCueModParent()
-	}
-
-	cueModDir := path.Join(p, "cue.mod")
-	if err := os.Mkdir(cueModDir, 0755); err != nil {
-		if !errors.Is(err, os.ErrExist) {
-			return err
-		}
-	}
-
-	if err := os.MkdirAll(cueModDir, 0755); err != nil {
-		return err
-	}
-
-	lockFilePath := path.Join(cueModDir, lockFilePath)
-	fileLock := flock.New(lockFilePath)
-	if err := fileLock.Lock(); err != nil {
-		return err
-	}
-
-	defer func() {
-		fileLock.Unlock()
-		os.Remove(lockFilePath)
-	}()
-
-	// ensure cue module is initialized
-	if err := cueModInit(ctx, p); err != nil {
-		return err
-	}
-
-	// add universe and lock file to `.gitignore`
-	if err := os.WriteFile(
-		path.Join(p, "cue.mod", "pkg", ".gitignore"),
-		[]byte(fmt.Sprintf("# generated by dagger\n%s\ndagger.lock\n", pkg.AlphaModule)),
-		0600,
-	); err != nil {
-		return err
-	}
-
-	log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
-
-	// Vendor in a temporary directory
-	tmp, err := os.MkdirTemp(path.Join(p, "cue.mod", "pkg"), "vendor-*")
-	if err != nil {
-		return err
-	}
-	if err := pkg.Vendor(ctx, tmp); err != nil {
-		// FIXME(samalba): disabled install remote stdlib temporarily
-		// if _, err := mod.Install(ctx, p, stdlib.ModuleName, ""); err != nil {
-		return err
-	}
-
-	// Semi-atomic swap of the vendor directory
-	// The following basically does:
-	// rm -rf cue.mod/pkg/MODULE.old
-	// mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old
-	// mv VENDOR cue.mod/pkg/MODULE
-	// rm -rf cue.mod/pkg/MODULE.old
-	newStdlib := path.Join(p, "cue.mod", "pkg", pkg.AlphaModule)
-	oldStdlib := newStdlib + ".old"
-	if err := os.RemoveAll(oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-	if err := os.Rename(newStdlib, oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-	defer os.RemoveAll(oldStdlib)
-
-	if err := os.Rename(path.Join(tmp, pkg.AlphaModule), newStdlib); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func getCueModParent() string {
-	cwd, _ := os.Getwd()
-	parentDir := cwd
-
-	// traverse the directory tree up through ancestors looking for a cue.mod folder
-	for {
-		if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) {
-			break // found it!
-		}
-
-		parentDir = filepath.Dir(parentDir)
-
-		if parentDir == string(os.PathSeparator) {
-			// reached the root
-			parentDir = cwd // reset to working directory
-			break
-		}
-	}
-
-	return parentDir
-}

state/state.go

@@ -6,6 +6,7 @@ import (
 
 	"cuelang.org/go/cue"
 	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/pkg"
 	"go.dagger.io/dagger/plancontext"
 )
 
@@ -55,7 +56,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
 	// 2) For backward compatibility: if the project was `dagger
 	// init`-ed before we added support for vendoring universe, it might not
 	// contain a `cue.mod`.
-	if err := VendorUniverse(ctx, w); err != nil {
+	if err := pkg.Vendor(ctx, w); err != nil {
 		return nil, err
 	}