Merge pull request #773 from aluzzardi/universe-vendoring

universe vendoring
This commit is contained in:
Andrea Luzzardi 2021-07-08 18:23:40 +02:00 committed by GitHub
commit 7fd80c774a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
158 changed files with 385 additions and 63 deletions

View File

@ -1,2 +0,0 @@
# dagger state
state/**

View File

@ -1,27 +0,0 @@
plan:
module: .dagger/env/test-core/plan
name: test-core
inputs:
dir:
dir:
path: ./tests
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhOW1rUy9vNFBDaXRDUHJu
ZVZYM1FucVorRDdUcmRaYmg3eTlTNzhhYWdjCndWZWxhZnhCZG4xZU9JQ1VyMXdR
OHY0TS81bk9FL2JuUEhuTmcxa29ORGcKLS0tIGxJUzNrZmRBNHZGRFY2Z01QK2JP
MlM1Ukdqbi9SQ0pqTi9FZ3MxN2E2QmsKHwd7P6KHPVdynOoto1jf3G4+5+vf87wU
HX1KD7Od5wRdBwn7r3OS8mdvuNIYpJDUb5YDrfjQypt020ohLocNiA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2021-06-17T14:07:53Z"
mac: ENC[AES256_GCM,data:afYut7wdvsJQgxlBg6NEV6DWk8vPi81mZgQAuWb4oe4WJeI1T9cYdtjOHPlmIpjqb86VQHJ29YTzektei2+k+VBawQxcsvefK7X1QboJTfMKLfsiug4qzNWjc7JZDvTb6dsDFM1U96gjSoAIVwdLMnucbu3681Fd7qSQgqNS61Q=,iv:ZQDHzXp0RJcUI4RtOVjdepV8zTa2kIHQhAltLkudDck=,tag:AnSFi1mKrEuXSqE4R+g7dw==,type:str]
pgp: []
encrypted_suffix: secret
version: 3.7.1

View File

@ -12,7 +12,6 @@ import (
"go.dagger.io/dagger/compiler" "go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/solver" "go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state" "go.dagger.io/dagger/state"
"go.dagger.io/dagger/stdlib"
"github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext" "github.com/opentracing/opentracing-go/ext"
@ -82,6 +81,18 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan") span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan")
defer span.Finish() defer span.Finish()
// FIXME: universe vendoring
// This is already done on `dagger init` and shouldn't be done here too.
// However:
// 1) As of right now, there's no way to update universe through the
// CLI, so we are lazily updating on `dagger up` using the embedded `universe`
// 2) For backward compatibility: if the workspace was `dagger
// init`-ed before we added support for vendoring universe, it might not
// contain a `cue.mod`.
if err := e.state.VendorUniverse(ctx); err != nil {
return err
}
planSource, err := e.state.Source().Compile("", e.state) planSource, err := e.state.Source().Compile("", e.state)
if err != nil { if err != nil {
return err return err
@ -95,8 +106,7 @@ func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
// Build a Cue config by overlaying the source with the stdlib // Build a Cue config by overlaying the source with the stdlib
sources := map[string]fs.FS{ sources := map[string]fs.FS{
stdlib.Path: stdlib.FS, "/": p.FS(),
"/": p.FS(),
} }
args := []string{} args := []string{}
if pkg := e.state.Plan.Package; pkg != "" { if pkg := e.state.Plan.Package; pkg != "" {

View File

@ -1,5 +1,10 @@
package state package state
import (
"context"
"path"
)
// Contents of an environment serialized to a file // Contents of an environment serialized to a file
type State struct { type State struct {
// State path // State path
@ -28,11 +33,26 @@ func (s *State) Source() Input {
w := s.Workspace w := s.Workspace
// FIXME: backward compatibility // FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" { if mod := s.Plan.Module; mod != "" {
w = mod w = path.Join(w, mod)
} }
return DirInput(w, []string{}, []string{}) return DirInput(w, []string{}, []string{})
} }
// VendorUniverse vendors the latest (built-in) version of the universe into the
// environment's `cue.mod`.
// FIXME: This has nothing to do in `State` and should be tied to a `Workspace`.
// However, since environments could point to different modules before, we have
// to handle vendoring on a per environment basis.
func (s *State) VendorUniverse(ctx context.Context) error {
w := s.Workspace
// FIXME: backward compatibility
if mod := s.Plan.Module; mod != "" {
w = path.Join(w, mod)
}
return vendorUniverse(ctx, w)
}
type Plan struct { type Plan struct {
Module string `yaml:"module,omitempty"` Module string `yaml:"module,omitempty"`
Package string `yaml:"package,omitempty"` Package string `yaml:"package,omitempty"`

View File

@ -12,6 +12,7 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"go.dagger.io/dagger/keychain" "go.dagger.io/dagger/keychain"
"go.dagger.io/dagger/stdlib"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@ -51,6 +52,11 @@ func Init(ctx context.Context, dir string) (*Workspace, error) {
if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil { if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil {
return nil, err return nil, err
} }
if err := vendorUniverse(ctx, root); err != nil {
return nil, err
}
return &Workspace{ return &Workspace{
Path: root, Path: root,
}, nil }, nil
@ -335,3 +341,63 @@ func (w *Workspace) cleanPackageName(ctx context.Context, pkg string) (string, e
return p, nil return p, nil
} }
func cueModInit(ctx context.Context, p string) error {
lg := log.Ctx(ctx)
mod := path.Join(p, "cue.mod")
if err := os.Mkdir(mod, 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
modFile := path.Join(mod, "module.cue")
if _, err := os.Stat(modFile); err != nil {
if !errors.Is(err, os.ErrNotExist) {
return err
}
lg.Debug().Str("mod", p).Msg("initializing cue.mod")
if err := os.WriteFile(modFile, []byte("module: \"\"\n"), 0600); err != nil {
return err
}
}
if err := os.Mkdir(path.Join(mod, "usr"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
if err := os.Mkdir(path.Join(mod, "pkg"), 0755); err != nil {
if !errors.Is(err, os.ErrExist) {
return err
}
}
return nil
}
func vendorUniverse(ctx context.Context, p string) error {
// ensure cue module is initialized
if err := cueModInit(ctx, p); err != nil {
return err
}
// add universe to `.gitignore`
if err := os.WriteFile(
path.Join(p, "cue.mod", "pkg", ".gitignore"),
[]byte(fmt.Sprintf("# dagger universe\n%s\n", stdlib.PackageName)),
0600,
); err != nil {
return err
}
log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
if err := stdlib.Vendor(ctx, p); err != nil {
return err
}
return nil
}

View File

@ -1,6 +1,5 @@
plan: plan:
module: ./aws/eks package: ./aws/eks/tests
package: ./tests
name: aws-eks name: aws-eks
inputs: inputs:
TestConfig.awsConfig.accessKey: TestConfig.awsConfig.accessKey:
@ -22,8 +21,8 @@ sops:
M3U4UFV5REQzYko3QjlXVE02Z0J4WUkK8uHC67Mutls4drXbCi8AwuFqbRXeb69P M3U4UFV5REQzYko3QjlXVE02Z0J4WUkK8uHC67Mutls4drXbCi8AwuFqbRXeb69P
ZnOFZEB4NoayoOojr1mY9ssDTywHF4KwR4E9ZmJ3V3hlEAgMkqfvSA== ZnOFZEB4NoayoOojr1mY9ssDTywHF4KwR4E9ZmJ3V3hlEAgMkqfvSA==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----
lastmodified: "2021-06-24T15:24:06Z" lastmodified: "2021-07-08T15:38:09Z"
mac: ENC[AES256_GCM,data:zRB7rhRuH95k2IxAbpf8FBfn0tiXKG4xEOv/M7qHxvexNhZWcBp1GnGnMI53CkzCMtTZcsdzquJJhQct2oyqXGbD9XNk9njC/hamtPVeXXtvDm2FTK8eHbb70tqIFCwspFXvfFMoNWCIZ5+CZTggTIOExnvAqsQ6QAN+FV3o25o=,iv:aghslE9r7hx1zmm2j/XznCyOJEoIs8CgUr5Wy/kuki0=,tag:6e701Vdjg+hnJElvZFnX7Q==,type:str] mac: ENC[AES256_GCM,data:S+MOmnBEpnWMQZPp6IFjPOMLWB6+OtgZOmUCwqivx0/I1IvVMiutHX+WI49ykGaZu0u8nKab8z/qnAwvw2lnUGMC5cE+UlB9ELZ4RcOO/KKOyEr6DTCchYEoEceuzQ+7aK+vVS5bx/IGIKCIVahTwz9anT/wiZDR2KgVRt/K0j0=,iv:T9msYqGXLoQmDUzVk0TUVjxxfr8+9Lp17cO7UkTSfkI=,tag:CWeu+7P7KRZ4kki3RLIZag==,type:str]
pgp: [] pgp: []
encrypted_suffix: secret encrypted_suffix: secret
version: 3.7.1 version: 3.7.1

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

2
stdlib/cue.mod/pkg/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -1,8 +1,13 @@
package stdlib package stdlib
import ( import (
"context"
"embed" "embed"
"fmt"
"io/fs"
"os"
"path" "path"
"path/filepath"
) )
var ( var (
@ -13,3 +18,38 @@ var (
PackageName = "alpha.dagger.io" PackageName = "alpha.dagger.io"
Path = path.Join("cue.mod", "pkg", PackageName) Path = path.Join("cue.mod", "pkg", PackageName)
) )
func Vendor(ctx context.Context, mod string) error {
// Remove any existing copy of the universe
if err := os.RemoveAll(path.Join(mod, Path)); err != nil {
return err
}
// Write the current version
return fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.Type().IsRegular() {
return nil
}
if filepath.Ext(entry.Name()) != ".cue" {
return nil
}
contents, err := fs.ReadFile(FS, p)
if err != nil {
return fmt.Errorf("%s: %w", p, err)
}
overlayPath := path.Join(mod, Path, p)
if err := os.MkdirAll(filepath.Dir(overlayPath), 0755); err != nil {
return err
}
return os.WriteFile(overlayPath, contents, 0600)
})
}

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -15,30 +15,34 @@ setup() {
# at the beginning of each new-style test. # at the beginning of each new-style test.
@test "core: inputs & outputs" { @test "core: inputs & outputs" {
# Use native Dagger environment here dagger init
unset DAGGER_WORKSPACE
# List available inputs dagger_new_with_plan test-core "$TESTDIR"/core/inputs-outputs
run dagger -e test-core input list
assert_success
assert_output --partial 'name'
assert_output --partial 'dir'
# Set text input # List available inputs
dagger -e test-core input text name Bob run dagger -e test-core input list
run dagger -e test-core up assert_success
assert_success assert_output --partial 'name'
assert_output --partial 'Hello, Bob!' assert_output --partial 'dir'
run dagger -e test-core output list # Set dir input
assert_success dagger -e test-core input dir dir "$DAGGER_WORKSPACE"
assert_output --partial 'message "Hello, Bob!"'
# Unset text input # Set text input
dagger -e test-core input unset name dagger -e test-core input text name Bob
run dagger -e test-core up run dagger -e test-core up
assert_success assert_success
assert_output --partial 'Hello, world!' assert_output --partial 'Hello, Bob!'
run dagger -e test-core output list
assert_success
assert_output --partial 'message "Hello, Bob!"'
# Unset text input
dagger -e test-core input unset name
run dagger -e test-core up
assert_success
assert_output --partial 'Hello, world!'
} }

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -1 +1 @@
module: "alpha.dagger.io/testing" module: "test.dagger.io/testing"

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -2,7 +2,7 @@ package testing
import ( import (
"alpha.dagger.io/dagger/op" "alpha.dagger.io/dagger/op"
"alpha.dagger.io/def" "test.dagger.io/def"
) )
#up: [ #up: [

View File

@ -1 +1 @@
module: "alpha.dagger.io/testing" module: "test.dagger.io/testing"

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -1,7 +1,7 @@
package testing package testing
import ( import (
"alpha.dagger.io/nonoptional" "test.dagger.io/nonoptional"
"alpha.dagger.io/dagger/op" "alpha.dagger.io/dagger/op"
) )

View File

@ -1 +1 @@
module: "alpha.dagger.io/testing" module: "test.dagger.io/testing"

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -1,7 +1,7 @@
package testing package testing
import ( import (
"alpha.dagger.io/optional" "test.dagger.io/optional"
"alpha.dagger.io/dagger/op" "alpha.dagger.io/dagger/op"
) )

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

View File

@ -0,0 +1 @@
module: ""

View File

@ -0,0 +1,2 @@
# dagger universe
alpha.dagger.io

Some files were not shown because too many files have changed in this diff Show More