performance: compile CUE client side

Restructured the compile logic to happen on the CLI instead of the
BuildKit frontend.

- Avoid uploading the entire workspace to BuildKit on every compilation
- Let the CUE loader scan the files instead of going through the
  BuildKit filesystem gRPC APIs.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi
2021-08-20 15:52:58 +02:00
parent 18c4978f07
commit b8dcc02bb8
9 changed files with 125 additions and 190 deletions

View File

@@ -3,7 +3,6 @@ package environment
import (
"context"
"fmt"
"io/fs"
"strings"
"time"
@@ -29,33 +28,38 @@ type Environment struct {
// Layer 2: user inputs
input *compiler.Value
// plan + inputs
src *compiler.Value
// Layer 3: computed values
computed *compiler.Value
}
func New(st *state.State) (*Environment, error) {
var err error
e := &Environment{
state: st,
plan: compiler.NewValue(),
input: compiler.NewValue(),
computed: compiler.NewValue(),
}
// Prepare inputs
for key, input := range st.Inputs {
v, err := input.Compile(key, st)
if err != nil {
return nil, err
}
if key == "" {
err = e.input.FillPath(cue.MakePath(), v)
} else {
err = e.input.FillPath(cue.ParsePath(key), v)
}
if err != nil {
return nil, err
}
e.plan, err = st.CompilePlan(context.TODO())
if err != nil {
return nil, err
}
e.input, err = st.CompileInputs()
if err != nil {
return nil, err
}
e.computed = compiler.NewValue()
e.src = compiler.NewValue()
if err := e.src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := e.src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}
return e, nil
@@ -65,64 +69,10 @@ func (e *Environment) Name() string {
return e.state.Name
}
func (e *Environment) Plan() *compiler.Value {
return e.plan
}
func (e *Environment) Input() *compiler.Value {
return e.input
}
func (e *Environment) Computed() *compiler.Value {
return e.computed
}
// LoadPlan loads the plan
func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
tr := otel.Tracer("environment")
ctx, span := tr.Start(ctx, "environment.LoadPlan")
defer span.End()
// FIXME: universe vendoring
// This is already done on `dagger init` and shouldn't be done here too.
// However:
// 1) As of right now, there's no way to update universe through the
// CLI, so we are lazily updating on `dagger up` using the embedded `universe`
// 2) For backward compatibility: if the workspace was `dagger
// init`-ed before we added support for vendoring universe, it might not
// contain a `cue.mod`.
if err := e.state.VendorUniverse(ctx); err != nil {
return err
}
planSource, err := e.state.Source().Compile("", e.state)
if err != nil {
return err
}
p := NewPipeline(planSource, s).WithCustomName("[internal] source")
// execute updater script
if err := p.Run(ctx); err != nil {
return err
}
// Build a Cue config by overlaying the source with the stdlib
sources := map[string]fs.FS{
"/": p.FS(),
}
args := []string{}
if pkg := e.state.Plan.Package; pkg != "" {
args = append(args, pkg)
}
plan, err := compiler.Build(sources, args...)
if err != nil {
return fmt.Errorf("plan config: %w", compiler.Err(err))
}
e.plan = plan
return nil
}
// Scan all scripts in the environment for references to local directories (do:"local"),
// and return all referenced directory names.
// This is used by clients to grant access to local directories when they are referenced
@@ -168,58 +118,33 @@ func (e *Environment) LocalDirs() map[string]string {
localdirs(v.Lookup("#up"))
}
// 2. Scan the plan
plan, err := e.state.Source().Compile("", e.state)
if err != nil {
panic(err)
}
localdirs(plan)
return dirs
}
// prepare initializes the Environment with inputs and plan code
func (e *Environment) prepare(ctx context.Context) (*compiler.Value, error) {
tr := otel.Tracer("environment")
_, span := tr.Start(ctx, "environment.Prepare")
defer span.End()
// Reset the computed values
e.computed = compiler.NewValue()
src := compiler.NewValue()
if err := src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}
return src, nil
}
// Up missing values in environment configuration, and write them to state.
func (e *Environment) Up(ctx context.Context, s solver.Solver) error {
tr := otel.Tracer("environment")
ctx, span := tr.Start(ctx, "environment.Up")
defer span.End()
// Set user inputs and plan code
src, err := e.prepare(ctx)
if err != nil {
return err
}
// Orchestrate execution with cueflow
flow := cueflow.New(
&cueflow.Config{},
src.Cue(),
e.src.Cue(),
newTaskFunc(newPipelineRunner(e.computed, s)),
)
if err := flow.Run(ctx); err != nil {
return err
}
return nil
// FIXME: canceling the context makes flow return `nil`
// Check explicitly if the context is canceled.
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
}
type DownOpts struct{}
@@ -328,20 +253,18 @@ func (e *Environment) ScanInputs(ctx context.Context, mergeUserInputs bool) ([]*
src := e.plan
if mergeUserInputs {
// Set user inputs and plan code
var err error
src, err = e.prepare(ctx)
if err != nil {
return nil, err
}
src = e.src
}
return ScanInputs(ctx, src), nil
}
func (e *Environment) ScanOutputs(ctx context.Context) ([]*compiler.Value, error) {
src, err := e.prepare(ctx)
if err != nil {
src := compiler.NewValue()
if err := src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}

View File

@@ -1,26 +0,0 @@
package environment
import (
"testing"
"github.com/stretchr/testify/require"
"go.dagger.io/dagger/state"
)
func TestLocalDirs(t *testing.T) {
st := &state.State{
Path: "/tmp/source",
Plan: state.Plan{
Module: "/tmp/source/plan",
},
}
require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{}, []string{})))
environment, err := New(st)
require.NoError(t, err)
localdirs := environment.LocalDirs()
require.Len(t, localdirs, 2)
require.Contains(t, localdirs, "/")
require.Contains(t, localdirs, "/tmp/source/plan")
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"cuelang.org/go/cue"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
)
@@ -43,13 +42,11 @@ func isReference(val cue.Value) bool {
}
func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
lg := log.Ctx(ctx)
inputs := []*compiler.Value{}
value.Walk(
func(val *compiler.Value) bool {
if isReference(val.Cue()) {
lg.Debug().Str("value.Path", val.Path().String()).Msg("found reference, stop walk")
return false
}
@@ -57,7 +54,6 @@ func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
return true
}
lg.Debug().Str("value.Path", val.Path().String()).Msg("found input")
inputs = append(inputs, val)
return true
@@ -68,7 +64,6 @@ func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
}
func ScanOutputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
lg := log.Ctx(ctx)
inputs := []*compiler.Value{}
value.Walk(
@@ -77,7 +72,6 @@ func ScanOutputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
return true
}
lg.Debug().Str("value.Path", val.Path().String()).Msg("found output")
inputs = append(inputs, val)
return true