kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

vendoring: support multiple modules

Browse Source 
Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2022-01-11 16:36:23 -08:00
parent 4e04bf9d22
commit fb16ac8b29
4 changed files with 153 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
pkg/pkg.go
View File

@ -3,24 +3,112 @@ package pkg
import (
"context"
"embed"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"github.com/gofrs/flock"
"github.com/rs/zerolog/log"
)
var (
// FS contains the filesystem of the stdlib.
//go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue alpha.dagger.io/europa/dagger/*.cue alpha.dagger.io/europa/dagger/engine/*.cue
//go:embed alpha.dagger.io/**/*.cue alpha.dagger.io/**/*/*.cue dagger.io/**/*.cue dagger.io/**/*/*.cue
FS embed.FS
AlphaModule = "alpha.dagger.io"
EnginePackage = fmt.Sprintf("%s/europa/dagger/engine", AlphaModule)
)
func Vendor(ctx context.Context, dest string) error {
// Write the current version
var (
AlphaModule = "alpha.dagger.io"
DaggerModule = "dagger.io"
EnginePackage = fmt.Sprintf("%s/dagger/engine", DaggerModule)
modules = []string{
AlphaModule,
DaggerModule,
}
lockFilePath = "dagger.lock"
)
func Vendor(ctx context.Context, p string) error {
if p == "" {
p = getCueModParent()
}
cuePkgDir := path.Join(p, "cue.mod", "pkg")
if err := os.MkdirAll(cuePkgDir, 0755); err != nil {
return err
}
// Lock this function so no more than 1 process can run it at once.
lockFile := path.Join(cuePkgDir, lockFilePath)
l := flock.New(lockFile)
if err := l.Lock(); err != nil {
return err
}
defer func() {
l.Unlock()
os.Remove(lockFile)
}()
// ensure cue module is initialized
if err := cueModInit(ctx, p); err != nil {
return err
}
// generate `.gitignore`
if err := os.WriteFile(
path.Join(cuePkgDir, ".gitignore"),
[]byte(fmt.Sprintf("# generated by dagger\n%s\n%s\ndagger.lock\n", AlphaModule, DaggerModule)),
0600,
); err != nil {
return err
}
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring packages")
// Unpack modules in a temporary directory
unpackDir, err := os.MkdirTemp(cuePkgDir, "vendor-*")
if err != nil {
return err
}
defer os.RemoveAll(unpackDir)
if err := extractModules(unpackDir); err != nil {
return err
}
for _, module := range modules {
// Semi-atomic swap of the module
//
// The following basically does:
// $ rm -rf cue.mod/pkg/MODULE.old
// $ mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old
// $ mv VENDOR/MODULE cue.mod/pkg/MODULE
// $ rm -rf cue.mod/pkg/MODULE.old
moduleDir := path.Join(cuePkgDir, module)
backupModuleDir := moduleDir + ".old"
if err := os.RemoveAll(backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := os.Rename(moduleDir, backupModuleDir); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
defer os.RemoveAll(backupModuleDir)
if err := os.Rename(path.Join(unpackDir, module), moduleDir); err != nil {
return err
}
}
return nil
}
func extractModules(dest string) error {
return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error {
if err != nil {
return err
@ -48,3 +136,56 @@ func Vendor(ctx context.Context, dest string) error {
return os.WriteFile(overlayPath, contents, 0600)
})
}
// getCueModParent traverses the directory tree up through ancestors looking for a cue.mod folder
func getCueModParent() string {
cwd, _ := os.Getwd()
parentDir := cwd
for {
if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) {
break // found it!
}
parentDir = filepath.Dir(parentDir)
if parentDir == string(os.PathSeparator) {
// reached the root
parentDir = cwd // reset to working directory
break
}
}
return parentDir
}
func cueModInit(ctx context.Context, parentDir string) error {
lg := log.Ctx(ctx)
modDir := path.Join(parentDir, "cue.mod")
modFile := path.Join(modDir, "module.cue")
if _, err := os.Stat(modFile); err != nil {
if !errors.Is(err, os.ErrNotExist) {
return err
}
lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
return err
}
}
if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
return nil
}

6
plan/plan.go
View File

@ -11,10 +11,10 @@ import (
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/pkg"
"go.dagger.io/dagger/plan/task"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state"
"go.opentelemetry.io/otel"
)
@ -33,8 +33,8 @@ type Config struct {
func Load(ctx context.Context, cfg Config) (*Plan, error) {
log.Ctx(ctx).Debug().Interface("args", cfg.Args).Msg("loading plan")
// FIXME: universe vendoring
if err := state.VendorUniverse(ctx, ""); err != nil {
// FIXME: vendoring path
if err := pkg.Vendor(ctx, ""); err != nil {
return nil, err
}

134
state/project.go
View File

@ -10,7 +10,6 @@ import (
"path/filepath"
"strings"
"github.com/gofrs/flock"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/keychain"
"go.dagger.io/dagger/pkg"
@ -32,7 +31,6 @@ const (
planDir = "plan"
manifestFile = "values.yaml"
computedFile = "computed.json"
lockFilePath = "dagger.lock"
)
type Project struct {
@ -57,7 +55,7 @@ func Init(ctx context.Context, dir string) (*Project, error) {
return nil, err
}
if err := VendorUniverse(ctx, root); err != nil {
if err := pkg.Vendor(ctx, root); err != nil {
return nil, err
}
@ -354,133 +352,3 @@ func (w *Project) cleanPackageName(ctx context.Context, pkg string) (string, err
return p, nil
}
func cueModInit(ctx context.Context, parentDir string) error {
lg := log.Ctx(ctx)
modDir := path.Join(parentDir, "cue.mod")
modFile := path.Join(modDir, "module.cue")
if _, err := os.Stat(modFile); err != nil {
if !errors.Is(err, os.ErrNotExist) {
return err
}
lg.Debug().Str("mod", parentDir).Msg("initializing cue.mod")
if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
return err
}
}
if err := os.Mkdir(path.Join(modDir, "usr"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
if err := os.Mkdir(path.Join(modDir, "pkg"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
return nil
}
func VendorUniverse(ctx context.Context, p string) error {
if p == "" {
p = getCueModParent()
}
cueModDir := path.Join(p, "cue.mod")
if err := os.Mkdir(cueModDir, 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
if err := os.MkdirAll(cueModDir, 0755); err != nil {
return err
}
lockFilePath := path.Join(cueModDir, lockFilePath)
fileLock := flock.New(lockFilePath)
if err := fileLock.Lock(); err != nil {
return err
}
defer func() {
fileLock.Unlock()
os.Remove(lockFilePath)
}()
// ensure cue module is initialized
if err := cueModInit(ctx, p); err != nil {
return err
}
// add universe and lock file to `.gitignore`
if err := os.WriteFile(
path.Join(p, "cue.mod", "pkg", ".gitignore"),
[]byte(fmt.Sprintf("# generated by dagger\n%s\ndagger.lock\n", pkg.AlphaModule)),
0600,
); err != nil {
return err
}
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
// Vendor in a temporary directory
tmp, err := os.MkdirTemp(path.Join(p, "cue.mod", "pkg"), "vendor-*")
if err != nil {
return err
}
if err := pkg.Vendor(ctx, tmp); err != nil {
// FIXME(samalba): disabled install remote stdlib temporarily
// if _, err := mod.Install(ctx, p, stdlib.ModuleName, ""); err != nil {
return err
}
// Semi-atomic swap of the vendor directory
// The following basically does:
// rm -rf cue.mod/pkg/MODULE.old
// mv cue.mod/pkg/MODULE cue.mod/pkg/MODULE.old
// mv VENDOR cue.mod/pkg/MODULE
// rm -rf cue.mod/pkg/MODULE.old
newStdlib := path.Join(p, "cue.mod", "pkg", pkg.AlphaModule)
oldStdlib := newStdlib + ".old"
if err := os.RemoveAll(oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := os.Rename(newStdlib, oldStdlib); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
defer os.RemoveAll(oldStdlib)
if err := os.Rename(path.Join(tmp, pkg.AlphaModule), newStdlib); err != nil {
return err
}
return nil
}
func getCueModParent() string {
cwd, _ := os.Getwd()
parentDir := cwd
// traverse the directory tree up through ancestors looking for a cue.mod folder
for {
if _, err := os.Stat(path.Join(parentDir, "cue.mod")); !errors.Is(err, os.ErrNotExist) {
break // found it!
}
parentDir = filepath.Dir(parentDir)
if parentDir == string(os.PathSeparator) {
// reached the root
parentDir = cwd // reset to working directory
break
}
}
return parentDir
}

3
state/state.go
View File

@ -6,6 +6,7 @@ import (
"cuelang.org/go/cue"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/pkg"
"go.dagger.io/dagger/plancontext"
)
@ -55,7 +56,7 @@ func (s *State) CompilePlan(ctx context.Context) (*compiler.Value, error) {
// 2) For backward compatibility: if the project was `dagger
// init`-ed before we added support for vendoring universe, it might not
// contain a `cue.mod`.
if err := VendorUniverse(ctx, w); err != nil {
if err := pkg.Vendor(ctx, w); err != nil {
return nil, err
}