From fb16ac8b299e516eaa6c0f0a2d8bc4b88064812d Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 11 Jan 2022 16:36:23 -0800 Subject: [PATCH] vendoring: support multiple modules Signed-off-by: Andrea Luzzardi --- pkg/pkg.go | 153 +++++++++++++++++++++++++++++++++++++++++++++-- plan/plan.go | 6 +- state/project.go | 134 +---------------------------------------- state/state.go | 3 +- 4 files changed, 153 insertions(+), 143 deletions(-) diff --git a/pkg/pkg.go b/pkg/pkg.go index 319b28c1..1b7ce4d6 100644 --- a/pkg/pkg.go +++ b/pkg/pkg.go @@ -3,24 +3,112 @@ package pkg import ( "context" "embed" + "errors" "fmt" "io/fs" "os" "path" "path/filepath" + + "github.com/gofrs/flock" + "github.com/rs/zerolog/log" ) var ( // FS contains the filesystem of the stdlib. - //go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue alpha.dagger.io/europa/dagger/*.cue alpha.dagger.io/europa/dagger/engine/*.cue + //go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue dagger.io/**/*.cue dagger.io/**/*/*.cue FS embed.FS - - AlphaModule = "alpha.dagger.io" - EnginePackage = fmt.Sprintf("%s/europa/dagger/engine", AlphaModule) ) -func Vendor(ctx context.Context, dest string) error { - // Write the current version +var ( + AlphaModule = "alpha.dagger.io" + DaggerModule = "dagger.io" + EnginePackage = fmt.Sprintf("%s/dagger/engine", DaggerModule) + + modules = []string{ + AlphaModule, + DaggerModule, + } + + lockFilePath = "dagger.lock" +) + +func Vendor(ctx context.Context, p string) error { + if p == "" { + p = getCueModParent() + } + + cuePkgDir := path.Join(p, "cue.mod", "pkg") + if err := os.MkdirAll(cuePkgDir, 0755); err != nil { + return err + } + + // Lock this function so no more than 1 process can run it at once. + lockFile := path.Join(cuePkgDir, lockFilePath) + l := flock.New(lockFile) + if err := l.Lock(); err != nil { + return err + } + defer func() { + l.Unlock() + os.Remove(lockFile) + }() + + // ensure cue module is initialized + if err := cueModInit(ctx, p); err != nil { + return err + } + + // generate `.gitignore` + if err := os.WriteFile( + path.Join(cuePkgDir, ".gitignore"), + []byte(fmt.Sprintf("# generated by dagger\n%s\n%s\ndagger.lock\n", AlphaModule, DaggerModule)), + 0600, + ); err != nil { + return err + } + + log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring packages") + + // Unpack modules in a temporary directory + unpackDir, err := os.MkdirTemp(cuePkgDir, "vendor-*") + if err != nil { + return err + } + defer os.RemoveAll(unpackDir) + + if err := extractModules(unpackDir); err != nil { + return err + } + + for _, module := range modules { + // Semi-atomic swap of the module + // + // The following basically does: + // $ rm -rf cue.mod/pkg/MODULE.old + // $ mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old + // $ mv VENDOR/MODULE cue.mod/pkg/MODULE + // $ rm -rf cue.mod/pkg/MODULE.old + + moduleDir := path.Join(cuePkgDir, module) + backupModuleDir := moduleDir + ".old" + if err := os.RemoveAll(backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.Rename(moduleDir, backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + defer os.RemoveAll(backupModuleDir) + + if err := os.Rename(path.Join(unpackDir, module), moduleDir); err != nil { + return err + } + } + + return nil +} + +func extractModules(dest string) error { return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error { if err != nil { return err @@ -48,3 +136,56 @@ func Vendor(ctx context.Context, dest string) error { return os.WriteFile(overlayPath, contents, 0600) }) } + +// getCueModParent traverses the directory tree up through ancestors looking for a cue.mod folder +func getCueModParent() string { + cwd, _ := os.Getwd() + parentDir := cwd + + for { + if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) { + break // found it! + } + + parentDir = filepath.Dir(parentDir) + + if parentDir == string(os.PathSeparator) { + // reached the root + parentDir = cwd // reset to working directory + break + } + } + + return parentDir +} + +func cueModInit(ctx context.Context, parentDir string) error { + lg := log.Ctx(ctx) + + modDir := path.Join(parentDir, "cue.mod") + modFile := path.Join(modDir, "module.cue") + if _, err := os.Stat(modFile); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + + lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod") + + if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil { + return err + } + } + + if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + } + if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + } + + return nil +} diff --git a/plan/plan.go b/plan/plan.go index 08346821..ffbed5d8 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -11,10 +11,10 @@ import ( "github.com/rs/zerolog/log" "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/environment" + "go.dagger.io/dagger/pkg" "go.dagger.io/dagger/plan/task" "go.dagger.io/dagger/plancontext" "go.dagger.io/dagger/solver" - "go.dagger.io/dagger/state" "go.opentelemetry.io/otel" ) @@ -33,8 +33,8 @@ type Config struct { func Load(ctx context.Context, cfg Config) (*Plan, error) { log.Ctx(ctx).Debug().Interface("args", cfg.Args).Msg("loading plan") - // FIXME: universe vendoring - if err := state.VendorUniverse(ctx, ""); err != nil { + // FIXME: vendoring path + if err := pkg.Vendor(ctx, ""); err != nil { return nil, err } diff --git a/state/project.go b/state/project.go index e035d69d..bc530d9e 100644 --- a/state/project.go +++ b/state/project.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strings" - "github.com/gofrs/flock" "github.com/rs/zerolog/log" "go.dagger.io/dagger/keychain" "go.dagger.io/dagger/pkg" @@ -32,7 +31,6 @@ const ( planDir = "plan" manifestFile = "values.yaml" computedFile = "computed.json" - lockFilePath = "dagger.lock" ) type Project struct { @@ -57,7 +55,7 @@ func Init(ctx context.Context, dir string) (*Project, error) { return nil, err } - if err := VendorUniverse(ctx, root); err != nil { + if err := pkg.Vendor(ctx, root); err != nil { return nil, err } @@ -354,133 +352,3 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err return p, nil } - -func cueModInit(ctx context.Context, parentDir string) error { - lg := log.Ctx(ctx) - - modDir := path.Join(parentDir, "cue.mod") - modFile := path.Join(modDir, "module.cue") - if _, err := os.Stat(modFile); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return err - } - - lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod") - - if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil { - return err - } - } - - if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil { - if !errors.Is(err, os.ErrExist) { - return err - } - } - if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil { - if !errors.Is(err, os.ErrExist) { - return err - } - } - - return nil -} - -func VendorUniverse(ctx context.Context, p string) error { - if p == "" { - p = getCueModParent() - } - - cueModDir := path.Join(p, "cue.mod") - if err := os.Mkdir(cueModDir, 0755); err != nil { - if !errors.Is(err, os.ErrExist) { - return err - } - } - - if err := os.MkdirAll(cueModDir, 0755); err != nil { - return err - } - - lockFilePath := path.Join(cueModDir, lockFilePath) - fileLock := flock.New(lockFilePath) - if err := fileLock.Lock(); err != nil { - return err - } - - defer func() { - fileLock.Unlock() - os.Remove(lockFilePath) - }() - - // ensure cue module is initialized - if err := cueModInit(ctx, p); err != nil { - return err - } - - // add universe and lock file to `.gitignore` - if err := os.WriteFile( - path.Join(p, "cue.mod", "pkg", ".gitignore"), - []byte(fmt.Sprintf("# generated by dagger\n%s\ndagger.lock\n", pkg.AlphaModule)), - 0600, - ); err != nil { - return err - } - - log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe") - - // Vendor in a temporary directory - tmp, err := os.MkdirTemp(path.Join(p, "cue.mod", "pkg"), "vendor-*") - if err != nil { - return err - } - if err := pkg.Vendor(ctx, tmp); err != nil { - // FIXME(samalba): disabled install remote stdlib temporarily - // if _, err := mod.Install(ctx, p, stdlib.ModuleName, ""); err != nil { - return err - } - - // Semi-atomic swap of the vendor directory - // The following basically does: - // rm -rf cue.mod/pkg/MODULE.old - // mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old - // mv VENDOR cue.mod/pkg/MODULE - // rm -rf cue.mod/pkg/MODULE.old - newStdlib := path.Join(p, "cue.mod", "pkg", pkg.AlphaModule) - oldStdlib := newStdlib + ".old" - if err := os.RemoveAll(oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - if err := os.Rename(newStdlib, oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - defer os.RemoveAll(oldStdlib) - - if err := os.Rename(path.Join(tmp, pkg.AlphaModule), newStdlib); err != nil { - return err - } - - return nil -} - -func getCueModParent() string { - cwd, _ := os.Getwd() - parentDir := cwd - - // traverse the directory tree up through ancestors looking for a cue.mod folder - for { - if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) { - break // found it! - } - - parentDir = filepath.Dir(parentDir) - - if parentDir == string(os.PathSeparator) { - // reached the root - parentDir = cwd // reset to working directory - break - } - } - - return parentDir -} diff --git a/state/state.go b/state/state.go index e68126e0..df68d2bd 100644 --- a/state/state.go +++ b/state/state.go @@ -6,6 +6,7 @@ import ( "cuelang.org/go/cue" "go.dagger.io/dagger/compiler" + "go.dagger.io/dagger/pkg" "go.dagger.io/dagger/plancontext" ) @@ -55,7 +56,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) { // 2) For backward compatibility: if the project was `dagger // init`-ed before we added support for vendoring universe, it might not // contain a `cue.mod`. - if err := VendorUniverse(ctx, w); err != nil { + if err := pkg.Vendor(ctx, w); err != nil { return nil, err }