kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Remove environment

Browse Source 
Signed-off-by: Joel Longtine <joel@dagger.io>
This commit is contained in:
Joel Longtine 2022-02-18 16:00:38 -07:00
parent c971077222
commit 034cd74ed0
10 changed files with 38 additions and 1735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
cmd/dagger/cmd/doc.go
View File

@ -20,7 +20,6 @@ import (
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/pkg"
"golang.org/x/term"
)
@ -55,23 +54,6 @@ type Package struct {
func Parse(ctx context.Context, packageName string, val *compiler.Value) *Package {
lg := log.Ctx(ctx)
parseValues := func(field string, values []*compiler.Value) []Value {
val := []Value{}
for _, i := range values {
v := Value{}
v.Name = strings.TrimPrefix(
i.Path().String(),
field+".",
)
v.Type = common.FormatValue(i)
v.Description = common.ValueDocOneLine(i)
val = append(val, v)
}
return val
}
fields, err := val.Fields(cue.Definitions(true))
if err != nil {
lg.Fatal().Err(err).Msg("cannot get fields")
@ -104,14 +86,6 @@ func Parse(ctx context.Context, packageName string, val *compiler.Value) *Packag
field.Name = name
field.Description = common.ValueDocOneLine(v)
// Inputs
inp := environment.ScanInputs(ctx, v)
field.Inputs = parseValues(field.Name, inp)
// Outputs
out := environment.ScanOutputs(ctx, v)
field.Outputs = parseValues(field.Name, out)
pkg.Fields = append(pkg.Fields, field)
}

106
cmd/dagger/cmd/up.go
View File

@ -2,19 +2,14 @@ package cmd
import (
"context"
"errors"
"fmt"
"os"
"cuelang.org/go/cue"
"go.dagger.io/dagger/client"
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/mod"
"go.dagger.io/dagger/plan"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
"golang.org/x/term"
@ -68,58 +63,6 @@ var upCmd = &cobra.Command{
return
}
project := common.CurrentProject(ctx)
st := common.CurrentEnvironmentState(ctx, project)
lg = lg.With().
Str("environment", st.Name).
Logger()
universeUpdateCh := make(chan bool)
go func() {
universeUpdateCh <- checkUniverseVersion(ctx, project.Path)
}()
doneCh := common.TrackProjectCommand(ctx, cmd, project, st)
env, err := environment.New(st)
if err != nil {
lg.Fatal().Err(err).Msg("unable to create environment")
}
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
// check that all inputs are set
if err := checkInputs(ctx, env); err != nil {
return err
}
if err := env.Up(ctx, s); err != nil {
return err
}
st.Computed = env.Computed().JSON().PrettyString()
if err := project.Save(ctx, st); err != nil {
return err
}
// FIXME: `ListOutput` is printing to Stdout directly which messes
// up the TTY logger.
if tty != nil {
tty.Stop()
}
return nil
})
<-doneCh
if err != nil {
lg.Fatal().Err(err).Msg("failed to up environment")
}
// Warn universe version if out of date
if update := <-universeUpdateCh; update {
fmt.Println("A new version of universe is available, please run 'dagger mod get alpha.dagger.io'")
}
},
}
@ -173,57 +116,8 @@ func europaUp(ctx context.Context, cl *client.Client, args ...string) error {
})
}
func checkInputs(ctx context.Context, env *environment.Environment) error {
lg := log.Ctx(ctx)
warnOnly := viper.GetBool("force")
notConcreteInputs := []*compiler.Value{}
inputs, err := env.ScanInputs(ctx, true)
if err != nil {
lg.Error().Err(err).Msg("failed to scan inputs")
return err
}
for _, i := range inputs {
isConcrete := (i.IsConcreteR(cue.Optional(true)) == nil)
switch {
case plancontext.IsSecretValue(i):
if _, err := env.Context().Secrets.FromValue(i); err != nil {
isConcrete = false
}
case plancontext.IsFSValue(i):
if _, err := env.Context().FS.FromValue(i); err != nil {
isConcrete = false
}
case plancontext.IsServiceValue(i):
if _, err := env.Context().Services.FromValue(i); err != nil {
isConcrete = false
}
}
if !isConcrete {
notConcreteInputs = append(notConcreteInputs, i)
}
}
for _, i := range notConcreteInputs {
if warnOnly {
lg.Warn().Str("input", i.Path().String()).Msg("required input is missing")
} else {
lg.Error().Str("input", i.Path().String()).Msg("required input is missing")
}
}
if !warnOnly && len(notConcreteInputs) > 0 {
return errors.New("some required inputs are not set, please re-run with `--force` if you think it's a mistake")
}
return nil
}
func init() {
upCmd.Flags().BoolP("force", "f", false, "Force up, disable inputs check")
upCmd.Flags().String("output", "", "Write computed output. Prints on stdout if set to-")
upCmd.Flags().StringArrayP("with", "w", []string{}, "")
upCmd.Flags().StringP("target", "t", "", "Run a single target of the DAG (for debugging only)")
upCmd.Flags().Bool("no-vendor", false, "Force up, disable inputs check")

42
cmd/dagger/logger/tty.go
View File

@ -14,14 +14,14 @@ import (
"github.com/containerd/console"
"github.com/morikuni/aec"
"github.com/tonistiigi/vt100"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/plan/task"
)
type Event map[string]interface{}
type Group struct {
Name string
State environment.State
State task.State
Events []Event
Started *time.Time
Completed *time.Time
@ -43,7 +43,7 @@ func (l *Logs) Add(event Event) error {
l.l.Lock()
defer l.l.Unlock()
task, ok := event["task"].(string)
taskPath, ok := event["task"].(string)
if !ok {
l.Messages = append(l.Messages, Message{
Event: event,
@ -52,12 +52,8 @@ func (l *Logs) Add(event Event) error {
return nil
}
// Hide `#up.*` from log group names
// FIXME: remove in Europa
groupKey := strings.Split(task, ".#up")[0]
// Hide hidden fields (e.g. `._*`) from log group names
groupKey = strings.Split(groupKey, "._")[0]
groupKey := strings.Split(taskPath, "._")[0]
group := l.groups[groupKey]
@ -82,8 +78,8 @@ func (l *Logs) Add(event Event) error {
// For each task in a group, the status will transition from computing to complete, then back to computing and so on.
// The transition is fast enough not to cause a problem.
if st, ok := event["state"].(string); ok {
group.State = environment.State(st)
if group.State == environment.StateComputing {
group.State = task.State(st)
if group.State == task.StateComputing {
group.Completed = nil
} else {
now := time.Now()
@ -228,7 +224,7 @@ func (c *TTYOutput) linesPerGroup(width, height int) int {
runningGroups := 0
for _, message := range c.logs.Messages {
if group := message.Group; group != nil && group.State == environment.StateComputing {
if group := message.Group; group != nil && group.State == task.StateComputing {
runningGroups++
}
}
@ -268,13 +264,13 @@ func (c *TTYOutput) printGroup(group *Group, width, maxLines int) int {
prefix := ""
switch group.State {
case environment.StateComputing:
case task.StateComputing:
prefix = "[+]"
case environment.StateCanceled:
case task.StateCanceled:
prefix = "[✗]"
case environment.StateFailed:
case task.StateFailed:
prefix = "[✗]"
case environment.StateCompleted:
case task.StateCompleted:
prefix = "[✔]"
}
@ -298,13 +294,13 @@ func (c *TTYOutput) printGroup(group *Group, width, maxLines int) int {
// color
switch group.State {
case environment.StateComputing:
case task.StateComputing:
out = aec.Apply(out, aec.LightBlueF)
case environment.StateCanceled:
case task.StateCanceled:
out = aec.Apply(out, aec.LightYellowF)
case environment.StateFailed:
case task.StateFailed:
out = aec.Apply(out, aec.LightRedF)
case environment.StateCompleted:
case task.StateCompleted:
out = aec.Apply(out, aec.LightGreenF)
}
@ -314,19 +310,19 @@ func (c *TTYOutput) printGroup(group *Group, width, maxLines int) int {
printEvents := []Event{}
switch group.State {
case environment.StateComputing:
case task.StateComputing:
printEvents = group.Events
// for computing tasks, show only last N
if len(printEvents) > maxLines {
printEvents = printEvents[len(printEvents)-maxLines:]
}
case environment.StateCanceled:
case task.StateCanceled:
// for completed tasks, don't show any logs
printEvents = []Event{}
case environment.StateFailed:
case task.StateFailed:
// for failed, show all logs
printEvents = group.Events
case environment.StateCompleted:
case task.StateCompleted:
// for completed tasks, don't show any logs
printEvents = []Event{}
}

212
environment/environment.go
View File

@ -1,212 +0,0 @@
package environment
import (
"context"
"fmt"
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/rs/zerolog/log"
)
type Environment struct {
state *state.State
// Layer 1: plan configuration
plan *compiler.Value
// Layer 2: user inputs
input *compiler.Value
// plan + inputs
src *compiler.Value
// Layer 3: computed values
computed *compiler.Value
}
func New(st *state.State) (*Environment, error) {
var err error
e := &Environment{
state: st,
}
e.plan, err = st.CompilePlan(context.TODO())
if err != nil {
return nil, err
}
e.input, err = st.CompileInputs()
if err != nil {
return nil, err
}
e.computed = compiler.NewValue()
e.src = compiler.NewValue()
if err := e.src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := e.src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}
return e, nil
}
func (e *Environment) Name() string {
return e.state.Name
}
func (e *Environment) Computed() *compiler.Value {
return e.computed
}
func (e *Environment) Context() *plancontext.Context {
return e.state.Context
}
// Up missing values in environment configuration, and write them to state.
func (e *Environment) Up(ctx context.Context, s solver.Solver) error {
ctx, span := otel.Tracer("dagger").Start(ctx, "environment.Up")
defer span.End()
// Orchestrate execution with cueflow
flow := cueflow.New(
&cueflow.Config{},
e.src.Cue(),
newTaskFunc(newPipelineRunner(e.computed, s, e.state.Context)),
)
if err := flow.Run(ctx); err != nil {
return err
}
// FIXME: canceling the context makes flow return `nil`
// Check explicitly if the context is canceled.
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
}
type DownOpts struct{}
func (e *Environment) Down(ctx context.Context, _ *DownOpts) error {
panic("NOT IMPLEMENTED")
}
type QueryOpts struct{}
func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc {
return func(flowVal cue.Value) (cueflow.Runner, error) {
v := compiler.Wrap(flowVal)
if !IsComponent(v) {
// No compute script
return nil, nil
}
return runner, nil
}
}
func newPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancontext.Context) cueflow.RunnerFunc {
return cueflow.RunnerFunc(func(t *cueflow.Task) error {
ctx := t.Context()
lg := log.
Ctx(ctx).
With().
Str("task", t.Path().String()).
Logger()
ctx = lg.WithContext(ctx)
ctx, span := otel.Tracer("dagger").Start(ctx, fmt.Sprintf("compute: %s", t.Path().String()))
defer span.End()
for _, dep := range t.Dependencies() {
lg.
Debug().
Str("dependency", dep.Path().String()).
Msg("dependency detected")
}
v := compiler.Wrap(t.Value())
p := NewPipeline(v, s, pctx)
err := p.Run(ctx)
if err != nil {
// Record the error
span.AddEvent("command", trace.WithAttributes(
attribute.String("error", err.Error()),
))
return err
}
// Mirror the computed values in both `Task` and `Result`
if !p.Computed().IsConcrete() {
return nil
}
if err := t.Fill(p.Computed().Cue()); err != nil {
lg.
Error().
Err(err).
Msg("failed to fill task")
return err
}
// Merge task value into output
if err := computed.FillPath(t.Path(), p.Computed()); err != nil {
lg.
Error().
Err(err).
Msg("failed to fill task result")
return err
}
return nil
})
}
func (e *Environment) ScanInputs(ctx context.Context, mergeUserInputs bool) ([]*compiler.Value, error) {
src := e.plan
if mergeUserInputs {
src = e.src
}
return ScanInputs(ctx, src), nil
}
func (e *Environment) ScanOutputs(ctx context.Context) ([]*compiler.Value, error) {
src := compiler.NewValue()
if err := src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}
if e.state.Computed != "" {
computed, err := compiler.DecodeJSON("", []byte(e.state.Computed))
if err != nil {
return nil, err
}
if err := src.FillPath(cue.MakePath(), computed); err != nil {
return nil, err
}
}
return ScanOutputs(ctx, src), nil
}

82
environment/inputs_scan.go
View File

@ -1,82 +0,0 @@
package environment
import (
"context"
"cuelang.org/go/cue"
"go.dagger.io/dagger/compiler"
)
func isReference(val cue.Value) bool {
isRef := func(v cue.Value) bool {
_, ref := v.ReferencePath()
if ref.String() == "" || v.Path().String() == ref.String() {
// not a reference
return false
}
for _, s := range ref.Selectors() {
if s.IsDefinition() {
// if we reference to a definition, we skip the check
return false
}
}
return true
}
op, vals := val.Expr()
if op == cue.NoOp {
return isRef(val)
}
for _, v := range vals {
// if the expr has an op (& or |, etc...), check the expr values, recursively
if isReference(v) {
return true
}
}
return isRef(val)
}
func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
inputs := []*compiler.Value{}
value.Walk(
func(val *compiler.Value) bool {
if isReference(val.Cue()) {
return false
}
if !val.HasAttr("input") {
return true
}
inputs = append(inputs, val)
return true
}, nil,
)
return inputs
}
func ScanOutputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
inputs := []*compiler.Value{}
value.Walk(
func(val *compiler.Value) bool {
if !val.HasAttr("output") {
return true
}
inputs = append(inputs, val)
return true
}, nil,
)
return inputs
}

1248
environment/pipeline.go
View File

File diff suppressed because it is too large Load Diff

9
plan/plan.go
View File

@ -10,7 +10,6 @@ import (
cueflow "cuelang.org/go/tools/flow"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/pkg"
"go.dagger.io/dagger/plan/task"
"go.dagger.io/dagger/plancontext"
@ -207,7 +206,7 @@ func newRunner(pctx *plancontext.Context, s solver.Solver, computed *compiler.Va
ctx, span := otel.Tracer("dagger").Start(ctx, fmt.Sprintf("up: %s", t.Path().String()))
defer span.End()
lg.Info().Str("state", string(environment.StateComputing)).Msg(string(environment.StateComputing))
lg.Info().Str("state", string(task.StateComputing)).Msg(string(task.StateComputing))
// Debug: dump dependencies
for _, dep := range t.Dependencies() {
@ -219,14 +218,14 @@ func newRunner(pctx *plancontext.Context, s solver.Solver, computed *compiler.Va
if err != nil {
// FIXME: this should use errdefs.IsCanceled(err)
if strings.Contains(err.Error(), "context canceled") {
lg.Error().Dur("duration", time.Since(start)).Str("state", string(environment.StateCanceled)).Msg(string(environment.StateCanceled))
lg.Error().Dur("duration", time.Since(start)).Str("state", string(task.StateCanceled)).Msg(string(task.StateCanceled))
} else {
lg.Error().Dur("duration", time.Since(start)).Err(err).Str("state", string(environment.StateFailed)).Msg(string(environment.StateFailed))
lg.Error().Dur("duration", time.Since(start)).Err(err).Str("state", string(task.StateFailed)).Msg(string(task.StateFailed))
}
return fmt.Errorf("%s: %w", t.Path().String(), err)
}
lg.Info().Dur("duration", time.Since(start)).Str("state", string(environment.StateCompleted)).Msg(string(environment.StateCompleted))
lg.Info().Dur("duration", time.Since(start)).Str("state", string(task.StateCompleted)).Msg(string(task.StateCompleted))
// If the result is not concrete (e.g. empty value), there's nothing to merge.
if !result.IsConcrete() {

27
plan/task/pipeline.go
View File

@ -1,27 +0,0 @@
package task
import (
"context"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
)
func init() {
Register("#up", func() Task { return &pipelineTask{} })
}
// pipelineTask is an adapter for legacy pipelines (`#up`).
// FIXME: remove once fully migrated to Europa.
type pipelineTask struct {
}
func (c *pipelineTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
p := environment.NewPipeline(v, s, pctx)
if err := p.Run(ctx); err != nil {
return nil, err
}
return p.Computed(), nil
}

17
plan/task/task.go
View File

@ -8,7 +8,6 @@ import (
"cuelang.org/go/cue"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/pkg"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
@ -23,6 +22,16 @@ var (
cue.Hid("_name", pkg.DaggerPackage))
)
// State is the state of the task.
type State string
const (
StateComputing = State("computing")
StateCanceled = State("canceled")
StateFailed = State("failed")
StateCompleted = State("completed")
)
type NewFunc func() Task
type Task interface {
@ -52,9 +61,9 @@ func New(typ string) Task {
func Lookup(v *compiler.Value) (Task, error) {
// FIXME: legacy pipelines
if environment.IsComponent(v) {
return New("#up"), nil
}
// if environment.IsComponent(v) {
// return New("#up"), nil
// }
if v.Kind() != cue.StructKind {
return nil, ErrNotTask

2
tests/tasks/pull/cue.mod/pkg/.gitignore vendored
View File

@ -1,5 +1,5 @@
# generated by dagger
dagger.lock
alpha.dagger.io
dagger.io
universe.dagger.io
dagger.lock