encode cue paths in LLB ops

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2021-02-23 15:37:32 -08:00
parent a9443bc0d2
commit 2ae395843e
3 changed files with 72 additions and 47 deletions

View File

@ -84,7 +84,7 @@ func (env *Env) SetInput(i *compiler.Value) error {
// Update the base configuration // Update the base configuration
func (env *Env) Update(ctx context.Context, s Solver) error { func (env *Env) Update(ctx context.Context, s Solver) error {
p := NewPipeline(s, nil) p := NewPipeline("[internal] source", s, nil)
// execute updater script // execute updater script
if err := p.Do(ctx, env.updater); err != nil { if err := p.Do(ctx, env.updater); err != nil {
return err return err
@ -229,7 +229,7 @@ func (env *Env) Compute(ctx context.Context, s Solver) error {
lg := lg. lg := lg.
With(). With().
Str("path", t.Path().String()). Str("component", t.Path().String()).
Str("state", t.State().String()). Str("state", t.State().String()).
Logger() Logger()
@ -284,7 +284,7 @@ func newPipelineTaskFunc(ctx context.Context, inst *cue.Instance, s Solver) cuef
lg := log. lg := log.
Ctx(ctx). Ctx(ctx).
With(). With().
Str("path", t.Path().String()). Str("component", t.Path().String()).
Logger() Logger()
ctx := lg.WithContext(ctx) ctx := lg.WithContext(ctx)
@ -295,7 +295,7 @@ func newPipelineTaskFunc(ctx context.Context, inst *cue.Instance, s Solver) cuef
Msg("dependency detected") Msg("dependency detected")
} }
v := compiler.Wrap(t.Value(), inst) v := compiler.Wrap(t.Value(), inst)
p := NewPipeline(s, NewFillable(t)) p := NewPipeline(t.Path().String(), s, NewFillable(t))
return p.Do(ctx, v) return p.Do(ctx, v)
}), nil }), nil
} }

View File

@ -32,6 +32,7 @@ func (fs FS) WriteValueJSON(filename string, v *compiler.Value) FS {
return fs.Change(func(st llb.State) llb.State { return fs.Change(func(st llb.State) llb.State {
return st.File( return st.File(
llb.Mkfile(filename, 0600, v.JSON()), llb.Mkfile(filename, 0600, v.JSON()),
llb.WithCustomName("[internal] serializing state to JSON"),
) )
}) })
} }
@ -44,6 +45,7 @@ func (fs FS) WriteValueCUE(filename string, v *compiler.Value) (FS, error) {
return fs.Change(func(st llb.State) llb.State { return fs.Change(func(st llb.State) llb.State {
return st.File( return st.File(
llb.Mkfile(filename, 0600, src), llb.Mkfile(filename, 0600, src),
llb.WithCustomName("[internal] serializing state to CUE"),
) )
}), nil }), nil
} }

View File

@ -19,13 +19,15 @@ import (
// An execution pipeline // An execution pipeline
type Pipeline struct { type Pipeline struct {
name string
s Solver s Solver
fs FS fs FS
out *Fillable out *Fillable
} }
func NewPipeline(s Solver, out *Fillable) *Pipeline { func NewPipeline(name string, s Solver, out *Fillable) *Pipeline {
return &Pipeline{ return &Pipeline{
name: name,
s: s, s: s,
fs: s.Scratch(), fs: s.Scratch(),
out: out, out: out,
@ -169,10 +171,16 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error {
} }
} }
func (p *Pipeline) vertexNamef(format string, a ...interface{}) string {
prefix := fmt.Sprintf("@%s@", p.name)
name := fmt.Sprintf(format, a...)
return prefix + " " + name
}
// Spawn a temporary pipeline with the same solver. // Spawn a temporary pipeline with the same solver.
// Output values are discarded: the parent pipeline's values are not modified. // Output values are discarded: the parent pipeline's values are not modified.
func (p *Pipeline) Tmp() *Pipeline { func (p *Pipeline) Tmp(name string) *Pipeline {
return NewPipeline(p.s, nil) return NewPipeline(name, p.s, nil)
} }
func (p *Pipeline) Subdir(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Subdir(ctx context.Context, op *compiler.Value) error {
@ -184,14 +192,17 @@ func (p *Pipeline) Subdir(ctx context.Context, op *compiler.Value) error {
return err return err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { p.fs = p.fs.Change(func(st llb.State) llb.State {
return st.File(llb.Copy( return st.File(
llb.Copy(
p.fs.LLB(), p.fs.LLB(),
dir, dir,
"/", "/",
&llb.CopyInfo{ &llb.CopyInfo{
CopyDirContentsOnly: true, CopyDirContentsOnly: true,
}, },
)) ),
llb.WithCustomName(p.vertexNamef("Subdir %s", dir)),
)
}) })
return nil return nil
} }
@ -207,12 +218,13 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value) error {
return err return err
} }
// Execute 'from' in a tmp pipeline, and use the resulting fs // Execute 'from' in a tmp pipeline, and use the resulting fs
from := p.Tmp() from := p.Tmp(op.Get("from").Path().String())
if err := from.Do(ctx, op.Get("from")); err != nil { if err := from.Do(ctx, op.Get("from")); err != nil {
return err return err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { p.fs = p.fs.Change(func(st llb.State) llb.State {
return st.File(llb.Copy( return st.File(
llb.Copy(
from.FS().LLB(), from.FS().LLB(),
src, src,
dest, dest,
@ -223,7 +235,9 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value) error {
CreateDestPath: true, CreateDestPath: true,
AllowWildcard: true, AllowWildcard: true,
}, },
)) ),
llb.WithCustomName(p.vertexNamef("Copy %s %s", src, dest)),
)
}) })
return nil return nil
} }
@ -241,11 +255,14 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value) error {
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { p.fs = p.fs.Change(func(st llb.State) llb.State {
return st.File(llb.Copy( return st.File(
llb.Copy(
llb.Local(dir, llb.FollowPaths(include)), llb.Local(dir, llb.FollowPaths(include)),
"/", "/",
"/", "/",
)) ),
llb.WithCustomName(p.vertexNamef("Local %s", dir)),
)
}) })
return nil return nil
} }
@ -262,9 +279,6 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
if err := op.Decode(&cmd); err != nil { if err := op.Decode(&cmd); err != nil {
return err return err
} }
// marker for status events
// FIXME
opts = append(opts, llb.WithCustomName(op.Path().String()))
// args // args
opts = append(opts, llb.Args(cmd.Args)) opts = append(opts, llb.Args(cmd.Args))
// dir // dir
@ -290,6 +304,11 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
} }
opts = append(opts, mntOpts...) opts = append(opts, mntOpts...)
} }
// marker for status events
// FIXME
opts = append(opts, llb.WithCustomName(p.vertexNamef("Exec %q", strings.Join(cmd.Args, " "))))
// --> Execute // --> Execute
p.fs = p.fs.Change(func(st llb.State) llb.State { p.fs = p.fs.Change(func(st llb.State) llb.State {
return st.Run(opts...).Root() return st.Run(opts...).Root()
@ -334,7 +353,7 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value)
} }
} }
// eg. mount: "/foo": { from: www.source } // eg. mount: "/foo": { from: www.source }
from := p.Tmp() from := p.Tmp(mnt.Get("from").Path().String())
if err := from.Do(ctx, mnt.Get("from")); err != nil { if err := from.Do(ctx, mnt.Get("from")); err != nil {
return nil, err return nil, err
} }
@ -434,7 +453,7 @@ func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) {
func (p *Pipeline) Load(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Load(ctx context.Context, op *compiler.Value) error {
// Execute 'from' in a tmp pipeline, and use the resulting fs // Execute 'from' in a tmp pipeline, and use the resulting fs
from := p.Tmp() from := p.Tmp(op.Get("from").Path().String())
if err := from.Do(ctx, op.Get("from")); err != nil { if err := from.Do(ctx, op.Get("from")); err != nil {
return err return err
} }
@ -449,7 +468,9 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value) error
return err return err
} }
// FIXME: preserve docker image metadata // FIXME: preserve docker image metadata
p.fs = p.fs.Set(llb.Image(ref)) p.fs = p.fs.Set(
llb.Image(ref, llb.WithCustomName(p.vertexNamef("FetchContainer %s", ref))),
)
return nil return nil
} }
@ -462,7 +483,9 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value) error {
if err != nil { if err != nil {
return err return err
} }
p.fs = p.fs.Set(llb.Git(remote, ref)) p.fs = p.fs.Set(
llb.Git(remote, ref, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remote, ref))),
)
return nil return nil
} }
@ -484,7 +507,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
// docker build context. This can come from another component, so we need to // docker build context. This can come from another component, so we need to
// compute it first. // compute it first.
if context.Exists() { if context.Exists() {
from := p.Tmp() from := p.Tmp(op.Lookup("context").Path().String())
if err := from.Do(ctx, context); err != nil { if err := from.Do(ctx, context); err != nil {
return err return err
} }