From f58ee5811b7ba2fa2d3bb503373b77372315c995 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 1 Jul 2021 19:42:52 +0200 Subject: [PATCH] universe vendoring Rather than injecting universe at runtime, this change will vendor alpha.dagger.io in `cue.mod` directly. Fixes #700 Signed-off-by: Andrea Luzzardi --- .dagger/env/test-core/.gitignore | 2 - .dagger/env/test-core/values.yaml | 27 -------- environment/environment.go | 16 ++++- state/state.go | 22 ++++++- state/workspace.go | 66 +++++++++++++++++++ stdlib/stdlib.go | 40 +++++++++++ tests/core.bats | 44 +++++++------ .../core/inputs-outputs}/test-core.cue | 0 .../undefined/with_pkg_def/cue.mod/module.cue | 2 +- .../def/main.cue | 0 .../ops/exec/undefined/with_pkg_def/main.cue | 2 +- .../with_pkg_mandatory/cue.mod/module.cue | 2 +- .../nonoptional/main.cue | 0 .../undefined/with_pkg_mandatory/main.cue | 2 +- .../with_pkg_optional/cue.mod/module.cue | 2 +- .../optional/main.cue | 0 .../exec/undefined/with_pkg_optional/main.cue | 2 +- 17 files changed, 170 insertions(+), 59 deletions(-) delete mode 100644 .dagger/env/test-core/.gitignore delete mode 100644 .dagger/env/test-core/values.yaml rename {.dagger/env/test-core/plan => tests/core/inputs-outputs}/test-core.cue (100%) rename tests/ops/exec/undefined/with_pkg_def/cue.mod/pkg/{alpha.dagger.io => test.dagger.io}/def/main.cue (100%) rename tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/pkg/{alpha.dagger.io => test.dagger.io}/nonoptional/main.cue (100%) rename tests/ops/exec/undefined/with_pkg_optional/cue.mod/pkg/{alpha.dagger.io => test.dagger.io}/optional/main.cue (100%) diff --git a/.dagger/env/test-core/.gitignore b/.dagger/env/test-core/.gitignore deleted file mode 100644 index 01ec19b0..00000000 --- a/.dagger/env/test-core/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# dagger state -state/** diff --git a/.dagger/env/test-core/values.yaml b/.dagger/env/test-core/values.yaml deleted file mode 100644 index 41ba947a..00000000 --- a/.dagger/env/test-core/values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -plan: - module: .dagger/env/test-core/plan -name: test-core -inputs: - dir: - dir: - path: ./tests -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhOW1rUy9vNFBDaXRDUHJu - ZVZYM1FucVorRDdUcmRaYmg3eTlTNzhhYWdjCndWZWxhZnhCZG4xZU9JQ1VyMXdR - OHY0TS81bk9FL2JuUEhuTmcxa29ORGcKLS0tIGxJUzNrZmRBNHZGRFY2Z01QK2JP - MlM1Ukdqbi9SQ0pqTi9FZ3MxN2E2QmsKHwd7P6KHPVdynOoto1jf3G4+5+vf87wU - HX1KD7Od5wRdBwn7r3OS8mdvuNIYpJDUb5YDrfjQypt020ohLocNiA== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2021-06-17T14:07:53Z" - mac: ENC[AES256_GCM,data:afYut7wdvsJQgxlBg6NEV6DWk8vPi81mZgQAuWb4oe4WJeI1T9cYdtjOHPlmIpjqb86VQHJ29YTzektei2+k+VBawQxcsvefK7X1QboJTfMKLfsiug4qzNWjc7JZDvTb6dsDFM1U96gjSoAIVwdLMnucbu3681Fd7qSQgqNS61Q=,iv:ZQDHzXp0RJcUI4RtOVjdepV8zTa2kIHQhAltLkudDck=,tag:AnSFi1mKrEuXSqE4R+g7dw==,type:str] - pgp: [] - encrypted_suffix: secret - version: 3.7.1 diff --git a/environment/environment.go b/environment/environment.go index 34ef1fc2..48d76fdc 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -12,7 +12,6 @@ import ( "go.dagger.io/dagger/compiler" "go.dagger.io/dagger/solver" "go.dagger.io/dagger/state" - "go.dagger.io/dagger/stdlib" "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -82,6 +81,18 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error { span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan") defer span.Finish() + // FIXME: universe vendoring + // This is already done on `dagger init` and shouldn't be done here too. + // However: + // 1) As of right now, there's no way to update universe through the + // CLI, so we are lazily updating on `dagger up` using the embedded `universe` + // 2) For backward compatibility: if the workspace was `dagger + // init`-ed before we added support for vendoring universe, it might not + // contain a `cue.mod`. + if err := e.state.VendorUniverse(ctx); err != nil { + return err + } + planSource, err := e.state.Source().Compile("", e.state) if err != nil { return err @@ -95,8 +106,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error { // Build a Cue config by overlaying the source with the stdlib sources := map[string]fs.FS{ - stdlib.Path: stdlib.FS, - "/": p.FS(), + "/": p.FS(), } args := []string{} if pkg := e.state.Plan.Package; pkg != "" { diff --git a/state/state.go b/state/state.go index b8c7a87a..16a5a9e5 100644 --- a/state/state.go +++ b/state/state.go @@ -1,5 +1,10 @@ package state +import ( + "context" + "path" +) + // Contents of an environment serialized to a file type State struct { // State path @@ -28,11 +33,26 @@ func (s *State) Source() Input { w := s.Workspace // FIXME: backward compatibility if mod := s.Plan.Module; mod != "" { - w = mod + w = path.Join(w, mod) } return DirInput(w, []string{}, []string{}) } +// VendorUniverse vendors the latest (built-in) version of the universe into the +// environment's `cue.mod`. +// FIXME: This has nothing to do in `State` and should be tied to a `Workspace`. +// However, since environments could point to different modules before, we have +// to handle vendoring on a per environment basis. +func (s *State) VendorUniverse(ctx context.Context) error { + w := s.Workspace + // FIXME: backward compatibility + if mod := s.Plan.Module; mod != "" { + w = path.Join(w, mod) + } + + return vendorUniverse(ctx, w) +} + type Plan struct { Module string `yaml:"module,omitempty"` Package string `yaml:"package,omitempty"` diff --git a/state/workspace.go b/state/workspace.go index dc7c4993..dacfddea 100644 --- a/state/workspace.go +++ b/state/workspace.go @@ -12,6 +12,7 @@ import ( "github.com/rs/zerolog/log" "go.dagger.io/dagger/keychain" + "go.dagger.io/dagger/stdlib" "gopkg.in/yaml.v3" ) @@ -51,6 +52,11 @@ func Init(ctx context.Context, dir string) (*Workspace, error) { if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil { return nil, err } + + if err := vendorUniverse(ctx, root); err != nil { + return nil, err + } + return &Workspace{ Path: root, }, nil @@ -335,3 +341,63 @@ func (w *Workspace) cleanPackageName(ctx context.Context, pkg string) (string, e return p, nil } + +func cueModInit(ctx context.Context, p string) error { + lg := log.Ctx(ctx) + + mod := path.Join(p, "cue.mod") + if err := os.Mkdir(mod, 0755); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + } + + modFile := path.Join(mod, "module.cue") + if _, err := os.Stat(modFile); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + + lg.Debug().Str("mod", p).Msg("initializing cue.mod") + + if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil { + return err + } + } + + if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + } + if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + } + + return nil +} + +func vendorUniverse(ctx context.Context, p string) error { + // ensure cue module is initialized + if err := cueModInit(ctx, p); err != nil { + return err + } + + // add universe to `.gitignore` + if err := os.WriteFile( + path.Join(p, "cue.mod", "pkg", ".gitignore"), + []byte(fmt.Sprintf("# dagger universe\n%s\n", stdlib.PackageName)), + 0600, + ); err != nil { + return err + } + + log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe") + if err := stdlib.Vendor(ctx, p); err != nil { + return err + } + + return nil +} diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index b2ac9019..f52eeb3a 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -1,8 +1,13 @@ package stdlib import ( + "context" "embed" + "fmt" + "io/fs" + "os" "path" + "path/filepath" ) var ( @@ -13,3 +18,38 @@ var ( PackageName = "alpha.dagger.io" Path = path.Join("cue.mod", "pkg", PackageName) ) + +func Vendor(ctx context.Context, mod string) error { + // Remove any existing copy of the universe + if err := os.RemoveAll(path.Join(mod, Path)); err != nil { + return err + } + + // Write the current version + return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !entry.Type().IsRegular() { + return nil + } + + if filepath.Ext(entry.Name()) != ".cue" { + return nil + } + + contents, err := fs.ReadFile(FS, p) + if err != nil { + return fmt.Errorf("%s: %w", p, err) + } + + overlayPath := path.Join(mod, Path, p) + + if err := os.MkdirAll(filepath.Dir(overlayPath), 0755); err != nil { + return err + } + + return os.WriteFile(overlayPath, contents, 0600) + }) +} diff --git a/tests/core.bats b/tests/core.bats index 4a636c4d..503bf7b5 100644 --- a/tests/core.bats +++ b/tests/core.bats @@ -15,30 +15,34 @@ setup() { # at the beginning of each new-style test. @test "core: inputs & outputs" { - # Use native Dagger environment here - unset DAGGER_WORKSPACE + dagger init - # List available inputs - run dagger -e test-core input list - assert_success - assert_output --partial 'name' - assert_output --partial 'dir' + dagger_new_with_plan test-core "$TESTDIR"/core/inputs-outputs - # Set text input - dagger -e test-core input text name Bob - run dagger -e test-core up - assert_success - assert_output --partial 'Hello, Bob!' + # List available inputs + run dagger -e test-core input list + assert_success + assert_output --partial 'name' + assert_output --partial 'dir' - run dagger -e test-core output list - assert_success - assert_output --partial 'message "Hello, Bob!"' + # Set dir input + dagger -e test-core input dir dir "$DAGGER_WORKSPACE" - # Unset text input - dagger -e test-core input unset name - run dagger -e test-core up - assert_success - assert_output --partial 'Hello, world!' + # Set text input + dagger -e test-core input text name Bob + run dagger -e test-core up + assert_success + assert_output --partial 'Hello, Bob!' + + run dagger -e test-core output list + assert_success + assert_output --partial 'message "Hello, Bob!"' + + # Unset text input + dagger -e test-core input unset name + run dagger -e test-core up + assert_success + assert_output --partial 'Hello, world!' } diff --git a/.dagger/env/test-core/plan/test-core.cue b/tests/core/inputs-outputs/test-core.cue similarity index 100% rename from .dagger/env/test-core/plan/test-core.cue rename to tests/core/inputs-outputs/test-core.cue diff --git a/tests/ops/exec/undefined/with_pkg_def/cue.mod/module.cue b/tests/ops/exec/undefined/with_pkg_def/cue.mod/module.cue index 7e830608..25575c7c 100644 --- a/tests/ops/exec/undefined/with_pkg_def/cue.mod/module.cue +++ b/tests/ops/exec/undefined/with_pkg_def/cue.mod/module.cue @@ -1 +1 @@ -module: "alpha.dagger.io/testing" +module: "test.dagger.io/testing" diff --git a/tests/ops/exec/undefined/with_pkg_def/cue.mod/pkg/alpha.dagger.io/def/main.cue b/tests/ops/exec/undefined/with_pkg_def/cue.mod/pkg/test.dagger.io/def/main.cue similarity index 100% rename from tests/ops/exec/undefined/with_pkg_def/cue.mod/pkg/alpha.dagger.io/def/main.cue rename to tests/ops/exec/undefined/with_pkg_def/cue.mod/pkg/test.dagger.io/def/main.cue diff --git a/tests/ops/exec/undefined/with_pkg_def/main.cue b/tests/ops/exec/undefined/with_pkg_def/main.cue index 50d175a9..052c25f9 100644 --- a/tests/ops/exec/undefined/with_pkg_def/main.cue +++ b/tests/ops/exec/undefined/with_pkg_def/main.cue @@ -2,7 +2,7 @@ package testing import ( "alpha.dagger.io/dagger/op" - "alpha.dagger.io/def" + "test.dagger.io/def" ) #up: [ diff --git a/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/module.cue b/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/module.cue index 7e830608..25575c7c 100644 --- a/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/module.cue +++ b/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/module.cue @@ -1 +1 @@ -module: "alpha.dagger.io/testing" +module: "test.dagger.io/testing" diff --git a/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/pkg/alpha.dagger.io/nonoptional/main.cue b/tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/pkg/test.dagger.io/nonoptional/main.cue similarity index 100% rename from tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/pkg/alpha.dagger.io/nonoptional/main.cue rename to tests/ops/exec/undefined/with_pkg_mandatory/cue.mod/pkg/test.dagger.io/nonoptional/main.cue diff --git a/tests/ops/exec/undefined/with_pkg_mandatory/main.cue b/tests/ops/exec/undefined/with_pkg_mandatory/main.cue index e2421bd1..0f1fd397 100644 --- a/tests/ops/exec/undefined/with_pkg_mandatory/main.cue +++ b/tests/ops/exec/undefined/with_pkg_mandatory/main.cue @@ -1,7 +1,7 @@ package testing import ( - "alpha.dagger.io/nonoptional" + "test.dagger.io/nonoptional" "alpha.dagger.io/dagger/op" ) diff --git a/tests/ops/exec/undefined/with_pkg_optional/cue.mod/module.cue b/tests/ops/exec/undefined/with_pkg_optional/cue.mod/module.cue index 7e830608..25575c7c 100644 --- a/tests/ops/exec/undefined/with_pkg_optional/cue.mod/module.cue +++ b/tests/ops/exec/undefined/with_pkg_optional/cue.mod/module.cue @@ -1 +1 @@ -module: "alpha.dagger.io/testing" +module: "test.dagger.io/testing" diff --git a/tests/ops/exec/undefined/with_pkg_optional/cue.mod/pkg/alpha.dagger.io/optional/main.cue b/tests/ops/exec/undefined/with_pkg_optional/cue.mod/pkg/test.dagger.io/optional/main.cue similarity index 100% rename from tests/ops/exec/undefined/with_pkg_optional/cue.mod/pkg/alpha.dagger.io/optional/main.cue rename to tests/ops/exec/undefined/with_pkg_optional/cue.mod/pkg/test.dagger.io/optional/main.cue diff --git a/tests/ops/exec/undefined/with_pkg_optional/main.cue b/tests/ops/exec/undefined/with_pkg_optional/main.cue index 1dbc5f88..3f12b3b0 100644 --- a/tests/ops/exec/undefined/with_pkg_optional/main.cue +++ b/tests/ops/exec/undefined/with_pkg_optional/main.cue @@ -1,7 +1,7 @@ package testing import ( - "alpha.dagger.io/optional" + "test.dagger.io/optional" "alpha.dagger.io/dagger/op" )