clean up & bugfix

Signed-off-by: Solomon Hykes <sh.github.6811@hykes.org>
This commit is contained in:
Solomon Hykes 2021-01-07 16:54:52 -08:00
parent 3162ca0991
commit c5842f894a
14 changed files with 166 additions and 129 deletions

View File

@ -30,7 +30,12 @@ func (c *Component) Config() *Value {
// NOTE: calling matchSpec("#Component") is not enough because // NOTE: calling matchSpec("#Component") is not enough because
// it does not match embedded scalars. // it does not match embedded scalars.
func (c Component) Validate() error { func (c Component) Validate() error {
return c.Config().Validate("#ComponentConfig") // FIXME: this crashes on `#dagger:compute:_`
// see TestValidateEmptyComponent
// Using a workaround for now.
// return c.Config().Validate("#ComponentConfig")
return c.Config().Validate()
} }
// Return this component's compute script. // Return this component's compute script.

43
dagger/component_test.go Normal file
View File

@ -0,0 +1,43 @@
package dagger
import (
"testing"
)
func TestValidateEmptyComponent(t *testing.T) {
cc := &Compiler{}
v, err := cc.Compile("", "#dagger: compute: _")
if err != nil {
t.Fatal(err)
}
_, err = v.Component()
if err != nil {
t.Fatal(err)
}
}
func TestValidateSimpleComponent(t *testing.T) {
cc := &Compiler{}
v, err := cc.Compile("", `hello: "world", #dagger: { compute: [{do:"local",dir:"foo"}]}`)
if err != nil {
t.Fatal(err)
}
c, err := v.Component()
if err != nil {
t.Fatal(err)
}
s, err := c.ComputeScript()
if err != nil {
t.Fatal(err)
}
n := 0
if err := s.Walk(func(op *Op) error {
n += 1
return nil
}); err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatal(s.v)
}
}

View File

@ -20,7 +20,8 @@ func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
} }
}() }()
debugf("initializing env") debugf("initializing env")
env, err := NewEnv(ctx, c) // Retrieve boot script form client
env, err := NewEnv(ctx, NewSolver(c), getBootScript(c), getInput(c))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -39,3 +40,17 @@ func Compute(ctx context.Context, c bkgw.Client) (r *bkgw.Result, err error) {
// Wrap cue directory in buildkit result // Wrap cue directory in buildkit result
return outdir.Result(ctx) return outdir.Result(ctx)
} }
func getBootScript(c bkgw.Client) string {
if boot, exists := c.BuildOpts().Opts["boot"]; exists {
return boot
}
return ""
}
func getInput(c bkgw.Client) string {
if input, exists := c.BuildOpts().Opts["input"]; exists {
return input
}
return ""
}

View File

@ -6,7 +6,6 @@ import (
"cuelang.org/go/cue" "cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow" cueflow "cuelang.org/go/tools/flow"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -28,27 +27,38 @@ type Env struct {
} }
// Initialize a new environment // Initialize a new environment
func NewEnv(ctx context.Context, c bkgw.Client) (*Env, error) { func NewEnv(ctx context.Context, s Solver, bootsrc, inputsrc string) (*Env, error) {
cc := &Compiler{} cc := &Compiler{}
// 1. Load base config (specified by client) // 1. Compile & execute boot script
debugf("Loading base configuration") boot, err := cc.CompileScript("boot.cue", bootsrc)
base, err := envBase(ctx, c, cc)
if err != nil { if err != nil {
return nil, err return nil, errors.Wrap(err, "compile boot script")
} }
// 2. Load input overlay (specified by client) bootfs, err := boot.Execute(ctx, s.Scratch(), Discard())
if err != nil {
return nil, errors.Wrap(err, "execute boot script")
}
// 2. load cue files produced by boot script
// FIXME: BuildAll() to force all files (no required package..)
debugf("building cue configuration from boot state")
base, err := cc.Build(ctx, bootfs)
if err != nil {
return nil, errors.Wrap(err, "load base config")
}
// 3. Compile & merge input overlay (user settings, input directories, secrets.)
debugf("Loading input overlay") debugf("Loading input overlay")
input, err := envInput(ctx, c, cc) input, err := cc.Compile("input.cue", inputsrc)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Check that input can be merged on base
if _, err := base.Merge(input); err != nil { if _, err := base.Merge(input); err != nil {
return nil, errors.Wrap(err, "merge base & input") return nil, errors.Wrap(err, "merge base & input")
} }
return &Env{ return &Env{
base: base, base: base,
input: input, input: input,
s: NewSolver(c), s: s,
cc: cc, cc: cc,
}, nil }, nil
} }
@ -57,6 +67,7 @@ func NewEnv(ctx context.Context, c bkgw.Client) (*Env, error) {
func (env *Env) Compute(ctx context.Context) error { func (env *Env) Compute(ctx context.Context) error {
debugf("Computing environment") debugf("Computing environment")
output, err := env.Walk(ctx, func(c *Component, out Fillable) error { output, err := env.Walk(ctx, func(c *Component, out Fillable) error {
debugf(" [Env.Compute] processing %s", c.Value().Path().String())
_, err := c.Compute(ctx, env.s, out) _, err := c.Compute(ctx, env.s, out)
return err return err
}) })
@ -95,6 +106,7 @@ func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
debugf("walking: \n----\n%s\n----\n", env.cc.Wrap(flowInst.Value(), flowInst).JSON())
// Initialize empty output // Initialize empty output
out, err := env.cc.EmptyStruct() out, err := env.cc.EmptyStruct()
if err != nil { if err != nil {
@ -125,6 +137,7 @@ func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
} }
// Cueflow match func // Cueflow match func
flowMatchFn := func(v cue.Value) (cueflow.Runner, error) { flowMatchFn := func(v cue.Value) (cueflow.Runner, error) {
debugf("Env.Walk: processing %s", v.Path().String())
val := env.cc.Wrap(v, flowInst) val := env.cc.Wrap(v, flowInst)
c, err := val.Component() c, err := val.Component()
if os.IsNotExist(err) { if os.IsNotExist(err) {
@ -145,42 +158,3 @@ func (env *Env) Walk(ctx context.Context, fn EnvWalkFunc) (*Value, error) {
} }
return out, nil return out, nil
} }
func envBase(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
// 1. Receive boot script from client.
debugf("retrieving boot script")
bootSrc, exists := c.BuildOpts().Opts["boot"]
if !exists {
// No boot script: return empty base config
return cc.EmptyStruct()
}
// 2. Compile & execute boot script
debugf("compiling boot script")
boot, err := cc.CompileScript("boot.cue", bootSrc)
if err != nil {
return nil, errors.Wrap(err, "compile boot script")
}
debugf("executing boot script")
bootState, err := boot.Execute(ctx, NewSolver(c).Scratch(), Discard())
if err != nil {
return nil, errors.Wrap(err, "execute boot script")
}
// 3. load cue files produced by bootstrap script
// FIXME: BuildAll() to force all files (no required package..)
debugf("building cue configuration from boot state")
base, err := cc.Build(ctx, bootState)
debugf("done building cue configuration: err=%q", err)
return base, err
}
func envInput(ctx context.Context, c bkgw.Client, cc *Compiler) (*Value, error) {
// 1. Receive input overlay from client.
// This is used to provide run-time settings, directories..
inputSrc, exists := c.BuildOpts().Opts["input"]
if !exists {
// No input overlay: return empty tree
return cc.EmptyStruct()
}
return cc.Compile("input.cue", inputSrc)
}

View File

@ -46,41 +46,8 @@ package dagger
// The contents of a #dagger annotation // The contents of a #dagger annotation
#ComponentConfig: { #ComponentConfig: {
// FIXME: deprecated
input?: bool
// script to compute the value // script to compute the value
compute?: #Script compute?: #Script
terminal?: {
// Display a message when opening a terminal session
greeting?: string
command: [string]: #Script
}
// Configure how the component is incorporated to user settings.
// Configure how the end-user can configure this component
settings?: {
// If not specified, scrape from comments
title?: string
description?: string
// Disable user input, even if incomplete?
hidden: true | *false
ui: _ // insert here something which can be compiled to react-jsonschema-form
// Show the cue default value to the user, as a default input value?
showDefault: true | *false
// Insert information needed by:
// 1) clients to encrypt
// ie. web wizard, cli
// 2) middleware to implement deicphering in the cuellb pipeline
// eg. integration with clcoud KMS, Vault...
//
// 3) connectors to make sure secrets are preserved
encrypt?: {
pubkey: string
cipher: string
}
}
} }
@ -92,7 +59,7 @@ package dagger
#Script: [...#Op] #Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load
// Export a value from fs state to cue // Export a value from fs state to cue
#Export: { #Export: {
@ -105,7 +72,7 @@ package dagger
#Local: { #Local: {
do: "local" do: "local"
dir: string dir: string
include: [...string] | *[] include?: [...string] | *[]
} }
// FIXME: bring back load (more efficient than copy) // FIXME: bring back load (more efficient than copy)

View File

@ -71,6 +71,7 @@ func (op *Op) Action() (Action, error) {
"#FetchContainer": op.FetchContainer, "#FetchContainer": op.FetchContainer,
"#FetchGit": op.FetchGit, "#FetchGit": op.FetchGit,
"#Local": op.Local, "#Local": op.Local,
"#Load": op.Load,
} }
for def, action := range actions { for def, action := range actions {
if err := op.Validate(def); err == nil { if err := op.Validate(def); err == nil {

View File

@ -4,6 +4,30 @@ import (
"testing" "testing"
) )
func TestLocalMatch(t *testing.T) {
cc := &Compiler{}
src := `do: "local", dir: "foo"`
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
op, err := v.Op()
if err != nil {
t.Fatal(err)
}
n := 0
err = op.Walk(func(op *Op) error {
n += 1
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatal(n)
}
}
func TestCopyMatch(t *testing.T) { func TestCopyMatch(t *testing.T) {
cc := &Compiler{} cc := &Compiler{}
src := `do: "copy", from: [{do: "local", dir: "foo"}]` src := `do: "copy", from: [{do: "local", dir: "foo"}]`

View File

@ -11,6 +11,7 @@ type Script struct {
} }
func (s Script) Validate() error { func (s Script) Validate() error {
// FIXME this crashes when a script is incomplete or empty
return s.Value().Validate("#Script") return s.Value().Validate("#Script")
} }

View File

@ -5,9 +5,37 @@ import (
"testing" "testing"
) )
func TestLocalScript(t *testing.T) {
cc := &Compiler{}
src := `[{do: "local", dir: "foo"}]`
v, err := cc.Compile("", src)
if err != nil {
t.Fatal(err)
}
s, err := v.Script()
if err != nil {
t.Fatal(err)
}
n := 0
err = s.Walk(func(op *Op) error {
n += 1
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatal(n)
}
}
func TestWalkBootScript(t *testing.T) { func TestWalkBootScript(t *testing.T) {
cc := &Compiler{} cc := &Compiler{}
script, err := cc.CompileScript("boot.cue", defaultBootScript) cfg, err := cc.Compile("clientconfig.cue", defaultBootScript)
if err != nil {
t.Fatal(err)
}
script, err := cfg.Get("bootscript").Script()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -41,41 +41,8 @@ package dagger
// The contents of a #dagger annotation // The contents of a #dagger annotation
#ComponentConfig: { #ComponentConfig: {
// FIXME: deprecated
input?: bool
// script to compute the value // script to compute the value
compute?: #Script compute?: #Script
terminal?: {
// Display a message when opening a terminal session
greeting?: string
command: [string]: #Script
}
// Configure how the component is incorporated to user settings.
// Configure how the end-user can configure this component
settings?: {
// If not specified, scrape from comments
title?: string
description?: string
// Disable user input, even if incomplete?
hidden: true | *false
ui: _ // insert here something which can be compiled to react-jsonschema-form
// Show the cue default value to the user, as a default input value?
showDefault: true | *false
// Insert information needed by:
// 1) clients to encrypt
// ie. web wizard, cli
// 2) middleware to implement deicphering in the cuellb pipeline
// eg. integration with clcoud KMS, Vault...
//
// 3) connectors to make sure secrets are preserved
encrypt?: {
pubkey: string
cipher: string
}
}
} }
@ -87,7 +54,7 @@ package dagger
#Script: [...#Op] #Script: [...#Op]
// One operation in a script // One operation in a script
#Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy #Op: #FetchContainer | #FetchGit | #Export | #Exec | #Local | #Copy | #Load
// Export a value from fs state to cue // Export a value from fs state to cue
#Export: { #Export: {
@ -100,7 +67,7 @@ package dagger
#Local: { #Local: {
do: "local" do: "local"
dir: string dir: string
include: [...string] | *[] include?: [...string] | *[]
} }
// FIXME: bring back load (more efficient than copy) // FIXME: bring back load (more efficient than copy)

View File

@ -18,15 +18,23 @@ func (s Spec) Validate(v *Value, defpath string) (err error) {
// FIXME: there is probably a cleaner way to do this. // FIXME: there is probably a cleaner way to do this.
defer func() { defer func() {
if err != nil { if err != nil {
debugf("ERROR while validating %v against %v err=%q", v, defpath, err)
err = fmt.Errorf("%s", cueerrors.Details(err, nil)) err = fmt.Errorf("%s", cueerrors.Details(err, nil))
} }
}() }()
def := s.root.LookupTarget(defpath) // Lookup def by name, eg. "#Script" or "#Copy"
if err := def.Err(); err != nil { // See dagger/spec.cue
def := s.root.Get(defpath)
if err := def.Validate(); err != nil {
return err return err
} }
if err := def.Unwrap().Fill(v).Validate(cue.Final()); err != nil { merged := def.Unwrap().Fill(v)
if err := merged.Err(); err != nil {
return err
}
debugf("Validating %v against %v", v, def)
if err := merged.Validate(cue.Final()); err != nil {
return err return err
} }
return nil return nil

View File

@ -312,9 +312,9 @@ func (v *Value) CueInst() *cue.Instance {
} }
func (v *Value) Compiler() *Compiler { func (v *Value) Compiler() *Compiler {
if v.cc == nil { // if v.cc == nil {
return &Compiler{} // return &Compiler{}
} // }
return v.cc return v.cc
} }

View File

@ -14,7 +14,11 @@ func TestSimple(t *testing.T) {
func TestCompileBootScript(t *testing.T) { func TestCompileBootScript(t *testing.T) {
cc := &Compiler{} cc := &Compiler{}
s, err := cc.CompileScript("boot.cue", defaultBootScript) cfg, err := cc.Compile("boot.cue", defaultBootScript)
if err != nil {
t.Fatal(err)
}
s, err := cfg.Get("bootscript").Script()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -16,7 +16,7 @@ let base=alpine & {
www: { www: {
source: { source: {
#dagger: input: true #dagger: compute: _
} }
host: string host: string