kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

cleanup: solver/fs

Browse Source 
- Solver: Encapsulates all access to Buildkit. Can solve plain LLB, invoke external frontends (for DockerBuild) and export (for ContainerPush)
- FS (now BuildkitFS) implements the standard Go 1.16 io/fs.FS interface and provides a read-only filesystem on top of a buildkit result. It can be used with built-ins such as fs.WalkDir (no need to have our own Walk functions anymore)
- Moved CueBuild into compiler.Build since it no longer depends on Buildkit. Instead it relies on the io/fs.FS interface, which is used both for the base config and the stdlib (go:embed also uses io/fs.FS). Overlaying base and the stdlib is now done by the same code.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2021-03-12 13:00:11 -08:00
parent c35eca99e1
commit c923e5042b
8 changed files with 365 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
dagger/build.go
View File

@ -1,67 +0,0 @@
package dagger
import (
"context"
"errors"
"fmt"
"path"
"path/filepath"
cueerrors "cuelang.org/go/cue/errors"
cueload "cuelang.org/go/cue/load"
"github.com/rs/zerolog/log"
"dagger.io/go/dagger/compiler"
"dagger.io/go/stdlib"
)
// Build a cue configuration tree from the files in fs.
func CueBuild(ctx context.Context, fs FS, args ...string) (*compiler.Value, error) {
var (
err error
lg = log.Ctx(ctx)
)
buildConfig := &cueload.Config{
// The CUE overlay needs to be prefixed by a non-conflicting path with the
// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
// files it finds locally.
Dir: "/config",
}
// Start by creating an overlay with the stdlib
buildConfig.Overlay, err = stdlib.Overlay(buildConfig.Dir)
if err != nil {
return nil, err
}
// Add the config files on top of the overlay
err = fs.Walk(ctx, func(p string, f Stat) error {
lg.Debug().Str("path", p).Msg("load")
if f.IsDir() {
return nil
}
if filepath.Ext(p) != ".cue" {
return nil
}
contents, err := fs.ReadFile(ctx, p)
if err != nil {
return fmt.Errorf("%s: %w", p, err)
}
overlayPath := path.Join(buildConfig.Dir, p)
buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents)
return nil
})
if err != nil {
return nil, err
}
instances := cueload.Instances(args, buildConfig)
if len(instances) != 1 {
return nil, errors.New("only one package is supported at a time")
}
inst, err := compiler.Cue().Build(instances[0])
if err != nil {
return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{}))
}
return compiler.Wrap(inst.Value(), inst), nil
}

17
dagger/client.go
View File

@ -20,6 +20,7 @@ import (
// buildkit // buildkit
bk "github.com/moby/buildkit/client" bk "github.com/moby/buildkit/client"
_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver _ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client" bkgw "github.com/moby/buildkit/frontend/gateway/client"
// docker output // docker output
@ -142,14 +143,22 @@ func (c *Client) buildfn(ctx context.Context, env *Env, ch chan *bk.SolveStatus,
} }
// Export env to a cue directory // Export env to a cue directory
// FIXME: this should be elsewhere
lg.Debug().Msg("exporting env") lg.Debug().Msg("exporting env")
outdir, err := env.Export(ctx, s.Scratch()) span, _ := opentracing.StartSpanFromContext(ctx, "Env.Export")
defer span.Finish()
st := llb.Scratch().File(
llb.Mkfile("state.cue", 0600, env.State().JSON()),
llb.WithCustomName("[internal] serializing state to JSON"),
)
ref, err := s.Solve(ctx, st)
if err != nil { if err != nil {
return nil, err return nil, err
} }
res := bkgw.NewResult()
// Wrap cue directory in buildkit result res.SetRef(ref)
return outdir.Result(ctx) return res, nil
}, ch) }, ch)
if err != nil { if err != nil {
return fmt.Errorf("buildkit solve: %w", bkCleanError(err)) return fmt.Errorf("buildkit solve: %w", bkCleanError(err))

63
dagger/compiler/build.go Normal file
View File

@ -0,0 +1,63 @@
package compiler
import (
"errors"
"fmt"
"io/fs"
"path"
"path/filepath"
cueerrors "cuelang.org/go/cue/errors"
cueload "cuelang.org/go/cue/load"
)
// Build a cue configuration tree from the files in fs.
func Build(sources map[string]fs.FS, args ...string) (*Value, error) {
buildConfig := &cueload.Config{
// The CUE overlay needs to be prefixed by a non-conflicting path with the
// local filesystem, otherwise Cue will merge the Overlay with whatever Cue
// files it finds locally.
Dir: "/config",
Overlay: map[string]cueload.Source{},
}
// Map the source files into the overlay
for mnt, f := range sources {
f := f
mnt := mnt
err := fs.WalkDir(f, ".", func(p string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.Type().IsRegular() {
return nil
}
if filepath.Ext(entry.Name()) != ".cue" {
return nil
}
contents, err := fs.ReadFile(f, p)
if err != nil {
return fmt.Errorf("%s: %w", p, err)
}
overlayPath := path.Join(buildConfig.Dir, mnt, p)
buildConfig.Overlay[overlayPath] = cueload.FromBytes(contents)
return nil
})
if err != nil {
return nil, err
}
}
instances := cueload.Instances(args, buildConfig)
if len(instances) != 1 {
return nil, errors.New("only one package is supported at a time")
}
inst, err := Cue().Build(instances[0])
if err != nil {
return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{}))
}
return Wrap(inst.Value(), inst), nil
}

38
dagger/env.go
View File

@ -3,12 +3,14 @@ package dagger
import ( import (
"context" "context"
"fmt" "fmt"
"io/fs"
"strings" "strings"
"time" "time"
"cuelang.org/go/cue" "cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow" cueflow "cuelang.org/go/tools/flow"
"dagger.io/go/dagger/compiler" "dagger.io/go/dagger/compiler"
"dagger.io/go/stdlib"
"github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext" "github.com/opentracing/opentracing-go/ext"
@ -98,9 +100,12 @@ func (env *Env) Update(ctx context.Context, s Solver) error {
return err return err
} }
// load cue files produced by updater // Build a Cue config by overlaying the source with the stdlib
// FIXME: BuildAll() to force all files (no required package..) sources := map[string]fs.FS{
base, err := CueBuild(ctx, p.FS()) stdlib.Path: stdlib.FS,
"/": p.FS(),
}
base, err := compiler.Build(sources)
if err != nil { if err != nil {
return fmt.Errorf("base config: %w", err) return fmt.Errorf("base config: %w", err)
} }
@ -190,33 +195,6 @@ func (env *Env) mergeState() error {
return nil return nil
} }
// Export env to a directory of cue files
// (Use with FS.Change)
func (env *Env) Export(ctx context.Context, fs FS) (FS, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "Env.Export")
defer span.Finish()
// FIXME: we serialize as JSON to guarantee a self-contained file.
// compiler.Value.Save() leaks imports, so requires a shared cue.mod with
// client which is undesirable.
// Once compiler.Value.Save() resolves non-builtin imports with a tree shake,
// we can use it here.
// FIXME: Exporting base/input/output separately causes merge errors.
// For instance, `foo: string | *"default foo"` gets serialized as
// `{"foo":"default foo"}`, which will fail to merge if output contains
// a different definition of `foo`.
//
// fs = env.base.SaveJSON(fs, "base.cue")
// fs = env.input.SaveJSON(fs, "input.cue")
// if env.output != nil {
// fs = env.output.SaveJSON(fs, "output.cue")
// }
// For now, export a single `state.cue` containing the combined output.
fs = fs.WriteValueJSON("state.cue", env.state)
return fs, nil
}
// Compute missing values in env configuration, and write them to state. // Compute missing values in env configuration, and write them to state.
func (env *Env) Compute(ctx context.Context, s Solver) error { func (env *Env) Compute(ctx context.Context, s Solver) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "Env.Compute") span, ctx := opentracing.StartSpanFromContext(ctx, "Env.Compute")

272
dagger/fs.go
View File

@ -3,228 +3,102 @@ package dagger
import ( import (
"context" "context"
"errors" "errors"
"os" "io/fs"
"path" "time"
"strings"
bk "github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client" bkgw "github.com/moby/buildkit/frontend/gateway/client"
bkpb "github.com/moby/buildkit/solver/pb"
fstypes "github.com/tonistiigi/fsutil/types" fstypes "github.com/tonistiigi/fsutil/types"
"dagger.io/go/dagger/compiler"
) )
type Stat struct { // BuildkitFS is a io/fs.FS adapter for Buildkit
*fstypes.Stat // BuildkitFS implements the ReadFileFS, StatFS and ReadDirFS interfaces.
type BuildkitFS struct {
ref bkgw.Reference
} }
type FS struct { func NewBuildkitFS(ref bkgw.Reference) *BuildkitFS {
// Before last solve return &BuildkitFS{
input llb.State ref: ref,
// After last solve }
output bkgw.Reference
// How to produce the output
s Solver
} }
func (fs FS) WriteValueJSON(filename string, v *compiler.Value) FS { // Open is not supported.
return fs.Change(func(st llb.State) llb.State { func (f *BuildkitFS) Open(name string) (fs.File, error) {
return st.File( return nil, errors.New("not implemented")
llb.Mkfile(filename, 0600, v.JSON()),
llb.WithCustomName("[internal] serializing state to JSON"),
)
})
} }
func (fs FS) WriteValueCUE(filename string, v *compiler.Value) (FS, error) { func (f *BuildkitFS) Stat(name string) (fs.FileInfo, error) {
src, err := v.Source() st, err := f.ref.StatFile(context.TODO(), bkgw.StatRequest{
if err != nil { Path: name,
return fs, err
}
return fs.Change(func(st llb.State) llb.State {
return st.File(
llb.Mkfile(filename, 0600, src),
llb.WithCustomName("[internal] serializing state to CUE"),
)
}), nil
}
func (fs FS) Solver() Solver {
return fs.s
}
// Compute output from input, if not done already.
// This method uses a pointer receiver to simplify
// calling it, since it is called in almost every
// other method.
func (fs *FS) solve(ctx context.Context) error {
if fs.output != nil {
return nil
}
output, err := fs.s.Solve(ctx, fs.input)
if err != nil {
return bkCleanError(err)
}
fs.output = output
return nil
}
func (fs FS) ReadFile(ctx context.Context, filename string) ([]byte, error) {
// Lazy solve
if err := (&fs).solve(ctx); err != nil {
return nil, err
}
// NOTE: llb.Scratch is represented by a `nil` reference. If solve result is
// Scratch, then `fs.output` is `nil`.
if fs.output == nil {
return nil, os.ErrNotExist
}
contents, err := fs.output.ReadFile(ctx, bkgw.ReadRequest{Filename: filename})
if err != nil {
return nil, bkCleanError(err)
}
return contents, nil
}
func (fs FS) ReadDir(ctx context.Context, dir string) ([]Stat, error) {
// Lazy solve
if err := (&fs).solve(ctx); err != nil {
return nil, err
}
// NOTE: llb.Scratch is represented by a `nil` reference. If solve result is
// Scratch, then `fs.output` is `nil`.
if fs.output == nil {
return []Stat{}, nil
}
st, err := fs.output.ReadDir(ctx, bkgw.ReadDirRequest{
Path: dir,
}) })
if err != nil { if err != nil {
return nil, bkCleanError(err) return nil, err
} }
out := make([]Stat, len(st)) return bkFileInfo{st}, nil
for i := range st {
out[i] = Stat{
Stat: st[i],
}
}
return out, nil
} }
func (fs FS) walk(ctx context.Context, p string, fn WalkFunc) error { func (f *BuildkitFS) ReadDir(name string) ([]fs.DirEntry, error) {
files, err := fs.ReadDir(ctx, p) entries, err := f.ref.ReadDir(context.TODO(), bkgw.ReadDirRequest{
if err != nil { Path: name,
return err })
}
for _, f := range files {
fPath := path.Join(p, f.GetPath())
if err := fn(fPath, f); err != nil {
return err
}
if f.IsDir() {
if err := fs.walk(ctx, fPath, fn); err != nil {
return err
}
}
}
return nil
}
type WalkFunc func(string, Stat) error
func (fs FS) Walk(ctx context.Context, fn WalkFunc) error {
return fs.walk(ctx, "/", fn)
}
type ChangeFunc func(llb.State) llb.State
func (fs FS) Change(changes ...ChangeFunc) FS {
for _, change := range changes {
fs = fs.Set(change(fs.input))
}
return fs
}
func (fs FS) Set(st llb.State) FS {
fs.input = st
fs.output = nil
return fs
}
func (fs FS) Solve(ctx context.Context) (FS, error) {
if err := (&fs).solve(ctx); err != nil {
return fs, err
}
return fs, nil
}
func (fs FS) LLB() llb.State {
return fs.input
}
func (fs FS) Def(ctx context.Context) (*bkpb.Definition, error) {
def, err := fs.LLB().Marshal(ctx, llb.LinuxAmd64)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return def.ToPB(), nil res := make([]fs.DirEntry, 0, len(entries))
} for _, st := range entries {
res = append(res, bkDirEntry{
func (fs FS) Ref(ctx context.Context) (bkgw.Reference, error) { bkFileInfo: bkFileInfo{
if err := (&fs).solve(ctx); err != nil { st: st,
return nil, err },
})
} }
return fs.output, nil
}
func (fs FS) Result(ctx context.Context) (*bkgw.Result, error) {
res := bkgw.NewResult()
ref, err := fs.Ref(ctx)
if err != nil {
return nil, err
}
res.SetRef(ref)
return res, nil return res, nil
} }
func (fs FS) Export(ctx context.Context, output bk.ExportEntry) (*bk.SolveResponse, error) { func (f *BuildkitFS) ReadFile(name string) ([]byte, error) {
// Lazy solve return f.ref.ReadFile(context.TODO(), bkgw.ReadRequest{
if err := (&fs).solve(ctx); err != nil { Filename: name,
return nil, err })
}
// NOTE: llb.Scratch is represented by a `nil` reference. If solve result is
// Scratch, then `fs.output` is `nil`.
if fs.output == nil {
return nil, os.ErrNotExist
}
st, err := fs.output.ToState()
if err != nil {
return nil, err
}
return fs.s.Export(ctx, st, output)
} }
// A helper to remove noise from buildkit error messages. // bkFileInfo is a fs.FileInfo adapter for fstypes.Stat
// FIXME: Obviously a cleaner solution would be nice. type bkFileInfo struct {
func bkCleanError(err error) error { st *fstypes.Stat
noise := []string{ }
"executor failed running ",
"buildkit-runc did not terminate successfully", func (s bkFileInfo) Name() string {
"rpc error: code = Unknown desc = ", return s.st.GetPath()
"failed to solve: ", }
}
func (s bkFileInfo) Size() int64 {
msg := err.Error() return s.st.GetSize_()
}
for _, s := range noise {
msg = strings.ReplaceAll(msg, s, "") func (s bkFileInfo) IsDir() bool {
} return s.st.IsDir()
}
return errors.New(msg)
func (s bkFileInfo) ModTime() time.Time {
return time.Unix(s.st.GetModTime(), 0)
}
func (s bkFileInfo) Mode() fs.FileMode {
return fs.FileMode(s.st.Mode)
}
func (s bkFileInfo) Sys() interface{} {
return s.st
}
// bkDirEntry is a fs.DirEntry adapter for fstypes.Stat
type bkDirEntry struct {
bkFileInfo
}
func (s bkDirEntry) Info() (fs.FileInfo, error) {
return s.bkFileInfo, nil
}
func (s bkDirEntry) Type() fs.FileMode {
return s.Mode()
} }

349
dagger/pipeline.go
View File

@ -22,23 +22,32 @@ import (
// An execution pipeline // An execution pipeline
type Pipeline struct { type Pipeline struct {
name string name string
s Solver s Solver
fs FS state llb.State
out *Fillable result bkgw.Reference
out *Fillable
} }
func NewPipeline(name string, s Solver, out *Fillable) *Pipeline { func NewPipeline(name string, s Solver, out *Fillable) *Pipeline {
return &Pipeline{ return &Pipeline{
name: name, name: name,
s: s, s: s,
fs: s.Scratch(), state: llb.Scratch(),
out: out, out: out,
} }
} }
func (p *Pipeline) FS() FS { func (p *Pipeline) State() llb.State {
return p.fs return p.state
}
func (p *Pipeline) Result() bkgw.Reference {
return p.result
}
func (p *Pipeline) FS() fs.FS {
return NewBuildkitFS(p.result)
} }
func isComponent(v *compiler.Value) bool { func isComponent(v *compiler.Value) bool {
@ -129,54 +138,54 @@ func (p *Pipeline) Do(ctx context.Context, code ...*compiler.Value) error {
Msg("pipeline was partially executed because of missing inputs") Msg("pipeline was partially executed because of missing inputs")
return nil return nil
} }
if err := p.doOp(ctx, op); err != nil { p.state, err = p.doOp(ctx, op, p.state)
if err != nil {
return err return err
} }
// Force a buildkit solve request at each operation, // Force a buildkit solve request at each operation,
// so that errors map to the correct cue path. // so that errors map to the correct cue path.
// FIXME: might as well change FS to make every operation // FIXME: might as well change FS to make every operation
// synchronous. // synchronous.
fs, err := p.fs.Solve(ctx) p.result, err = p.s.Solve(ctx, p.state)
if err != nil { if err != nil {
return err return err
} }
p.fs = fs
} }
return nil return nil
} }
func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
do, err := op.Get("do").String() do, err := op.Get("do").String()
if err != nil { if err != nil {
return err return st, err
} }
switch do { switch do {
case "copy": case "copy":
return p.Copy(ctx, op) return p.Copy(ctx, op, st)
case "exec": case "exec":
return p.Exec(ctx, op) return p.Exec(ctx, op, st)
case "export": case "export":
return p.Export(ctx, op) return p.Export(ctx, op, st)
case "fetch-container": case "fetch-container":
return p.FetchContainer(ctx, op) return p.FetchContainer(ctx, op, st)
case "push-container": case "push-container":
return p.PushContainer(ctx, op) return p.PushContainer(ctx, op, st)
case "fetch-git": case "fetch-git":
return p.FetchGit(ctx, op) return p.FetchGit(ctx, op, st)
case "local": case "local":
return p.Local(ctx, op) return p.Local(ctx, op, st)
case "load": case "load":
return p.Load(ctx, op) return p.Load(ctx, op, st)
case "subdir": case "subdir":
return p.Subdir(ctx, op) return p.Subdir(ctx, op, st)
case "docker-build": case "docker-build":
return p.DockerBuild(ctx, op) return p.DockerBuild(ctx, op, st)
case "write-file": case "write-file":
return p.WriteFile(ctx, op) return p.WriteFile(ctx, op, st)
case "mkdir": case "mkdir":
return p.Mkdir(ctx, op) return p.Mkdir(ctx, op, st)
default: default:
return fmt.Errorf("invalid operation: %s", op.JSON()) return st, fmt.Errorf("invalid operation: %s", op.JSON())
} }
} }
@ -192,74 +201,68 @@ func (p *Pipeline) Tmp(name string) *Pipeline {
return NewPipeline(name, p.s, nil) return NewPipeline(name, p.s, nil)
} }
func (p *Pipeline) Subdir(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Subdir(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
// FIXME: this could be more optimized by carrying subdir path as metadata, // FIXME: this could be more optimized by carrying subdir path as metadata,
// and using it in copy, load or mount. // and using it in copy, load or mount.
dir, err := op.Get("dir").String() dir, err := op.Get("dir").String()
if err != nil { if err != nil {
return err return st, err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.File(
return st.File( llb.Copy(
llb.Copy( st,
p.fs.LLB(), dir,
dir, "/",
"/", &llb.CopyInfo{
&llb.CopyInfo{ CopyDirContentsOnly: true,
CopyDirContentsOnly: true, },
}, ),
), llb.WithCustomName(p.vertexNamef("Subdir %s", dir)),
llb.WithCustomName(p.vertexNamef("Subdir %s", dir)), ), nil
)
})
return nil
} }
func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
// Decode copy options // Decode copy options
src, err := op.Get("src").String() src, err := op.Get("src").String()
if err != nil { if err != nil {
return err return st, err
} }
dest, err := op.Get("dest").String() dest, err := op.Get("dest").String()
if err != nil { if err != nil {
return err return st, err
} }
// Execute 'from' in a tmp pipeline, and use the resulting fs // Execute 'from' in a tmp pipeline, and use the resulting fs
from := p.Tmp(op.Get("from").Path().String()) from := p.Tmp(op.Get("from").Path().String())
if err := from.Do(ctx, op.Get("from")); err != nil { if err := from.Do(ctx, op.Get("from")); err != nil {
return err return st, err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.File(
return st.File( llb.Copy(
llb.Copy( from.State(),
from.FS().LLB(), src,
src, dest,
dest, // FIXME: allow more configurable llb options
// FIXME: allow more configurable llb options // For now we define the following convenience presets:
// For now we define the following convenience presets: &llb.CopyInfo{
&llb.CopyInfo{ CopyDirContentsOnly: true,
CopyDirContentsOnly: true, CreateDestPath: true,
CreateDestPath: true, AllowWildcard: true,
AllowWildcard: true, },
}, ),
), llb.WithCustomName(p.vertexNamef("Copy %s %s", src, dest)),
llb.WithCustomName(p.vertexNamef("Copy %s %s", src, dest)), ), nil
)
})
return nil
} }
func (p *Pipeline) Local(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
dir, err := op.Get("dir").String() dir, err := op.Get("dir").String()
if err != nil { if err != nil {
return err return st, err
} }
var include []string var include []string
if inc := op.Get("include"); inc.Exists() { if inc := op.Get("include"); inc.Exists() {
if err := inc.Decode(&include); err != nil { if err := inc.Decode(&include); err != nil {
return err return st, err
} }
} }
// FIXME: Remove the `Copy` and use `Local` directly. // FIXME: Remove the `Copy` and use `Local` directly.
@ -270,30 +273,26 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value) error {
// //
// By wrapping `llb.Local` inside `llb.Copy`, we get the same digest for // By wrapping `llb.Local` inside `llb.Copy`, we get the same digest for
// the same content. // the same content.
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.File(
return st.File( llb.Copy(
llb.Copy( llb.Local(
llb.Local( dir,
dir, llb.FollowPaths(include),
llb.FollowPaths(include), llb.WithCustomName(p.vertexNamef("Local %s [transfer]", dir)),
llb.WithCustomName(p.vertexNamef("Local %s [transfer]", dir)),
// Without hint, multiple `llb.Local` operations on the // Without hint, multiple `llb.Local` operations on the
// same path get a different digest. // same path get a different digest.
llb.SessionID(p.s.SessionID()), llb.SessionID(p.s.SessionID()),
llb.SharedKeyHint(dir), llb.SharedKeyHint(dir),
),
"/",
"/",
), ),
llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)), "/",
) "/",
}) ),
llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)),
return nil ), nil
} }
func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
opts := []llb.RunOption{} opts := []llb.RunOption{}
var cmd struct { var cmd struct {
Args []string Args []string
@ -303,7 +302,7 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
} }
if err := op.Decode(&cmd); err != nil { if err := op.Decode(&cmd); err != nil {
return err return st, err
} }
// args // args
opts = append(opts, llb.Args(cmd.Args)) opts = append(opts, llb.Args(cmd.Args))
@ -318,7 +317,7 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
if cmd.Always { if cmd.Always {
cacheBuster, err := randomID(8) cacheBuster, err := randomID(8)
if err != nil { if err != nil {
return err return st, err
} }
opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster)) opts = append(opts, llb.AddEnv("DAGGER_CACHEBUSTER", cacheBuster))
} }
@ -326,7 +325,7 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
if mounts := op.Lookup("mount"); mounts.Exists() { if mounts := op.Lookup("mount"); mounts.Exists() {
mntOpts, err := p.mountAll(ctx, mounts) mntOpts, err := p.mountAll(ctx, mounts)
if err != nil { if err != nil {
return err return st, err
} }
opts = append(opts, mntOpts...) opts = append(opts, mntOpts...)
} }
@ -340,10 +339,7 @@ func (p *Pipeline) Exec(ctx context.Context, op *compiler.Value) error {
opts = append(opts, llb.WithCustomName(p.vertexNamef("Exec [%s]", strings.Join(args, ", ")))) opts = append(opts, llb.WithCustomName(p.vertexNamef("Exec [%s]", strings.Join(args, ", "))))
// --> Execute // --> Execute
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.Run(opts...).Root(), nil
return st.Run(opts...).Root()
})
return nil
} }
func (p *Pipeline) mountAll(ctx context.Context, mounts *compiler.Value) ([]llb.RunOption, error) { func (p *Pipeline) mountAll(ctx context.Context, mounts *compiler.Value) ([]llb.RunOption, error) {
@ -397,21 +393,21 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value)
} }
mo = append(mo, llb.SourcePath(mps)) mo = append(mo, llb.SourcePath(mps))
} }
return llb.AddMount(dest, from.FS().LLB(), mo...), nil return llb.AddMount(dest, from.State(), mo...), nil
} }
func (p *Pipeline) Export(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Export(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
source, err := op.Get("source").String() source, err := op.Get("source").String()
if err != nil { if err != nil {
return err return st, err
} }
format, err := op.Get("format").String() format, err := op.Get("format").String()
if err != nil { if err != nil {
return err return st, err
} }
contents, err := p.fs.ReadFile(ctx, source) contents, err := fs.ReadFile(p.FS(), source)
if err != nil { if err != nil {
return fmt.Errorf("export %s: %w", source, err) return st, fmt.Errorf("export %s: %w", source, err)
} }
switch format { switch format {
case "string": case "string":
@ -422,13 +418,13 @@ func (p *Pipeline) Export(ctx context.Context, op *compiler.Value) error {
Msg("exporting string") Msg("exporting string")
if err := p.out.Fill(string(contents)); err != nil { if err := p.out.Fill(string(contents)); err != nil {
return err return st, err
} }
case "json": case "json":
var o interface{} var o interface{}
o, err := unmarshalAnything(contents, json.Unmarshal) o, err := unmarshalAnything(contents, json.Unmarshal)
if err != nil { if err != nil {
return err return st, err
} }
log. log.
@ -438,13 +434,13 @@ func (p *Pipeline) Export(ctx context.Context, op *compiler.Value) error {
Msg("exporting json") Msg("exporting json")
if err := p.out.Fill(o); err != nil { if err := p.out.Fill(o); err != nil {
return err return st, err
} }
case "yaml": case "yaml":
var o interface{} var o interface{}
o, err := unmarshalAnything(contents, yaml.Unmarshal) o, err := unmarshalAnything(contents, yaml.Unmarshal)
if err != nil { if err != nil {
return err return st, err
} }
log. log.
@ -454,12 +450,12 @@ func (p *Pipeline) Export(ctx context.Context, op *compiler.Value) error {
Msg("exporting yaml") Msg("exporting yaml")
if err := p.out.Fill(o); err != nil { if err := p.out.Fill(o); err != nil {
return err return st, err
} }
default: default:
return fmt.Errorf("unsupported export format: %q", format) return st, fmt.Errorf("unsupported export format: %q", format)
} }
return nil return st, nil
} }
type unmarshaller func([]byte, interface{}) error type unmarshaller func([]byte, interface{}) error
@ -481,31 +477,30 @@ func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) {
return o, err return o, err
} }
func (p *Pipeline) Load(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
// Execute 'from' in a tmp pipeline, and use the resulting fs // Execute 'from' in a tmp pipeline, and use the resulting fs
from := p.Tmp(op.Get("from").Path().String()) from := p.Tmp(op.Get("from").Path().String())
if err := from.Do(ctx, op.Get("from")); err != nil { if err := from.Do(ctx, op.Get("from")); err != nil {
return err return st, err
} }
p.fs = p.fs.Set(from.FS().LLB()) return from.State(), nil
return nil
} }
func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
rawRef, err := op.Get("ref").String() rawRef, err := op.Get("ref").String()
if err != nil { if err != nil {
return err return st, err
} }
ref, err := reference.ParseNormalizedNamed(rawRef) ref, err := reference.ParseNormalizedNamed(rawRef)
if err != nil { if err != nil {
return fmt.Errorf("failed to parse ref %s: %w", rawRef, err) return st, fmt.Errorf("failed to parse ref %s: %w", rawRef, err)
} }
// Add the default tag "latest" to a reference if it only has a repo name. // Add the default tag "latest" to a reference if it only has a repo name.
ref = reference.TagNameOnly(ref) ref = reference.TagNameOnly(ref)
state := llb.Image( st = llb.Image(
ref.String(), ref.String(),
llb.WithCustomName(p.vertexNamef("FetchContainer %s", rawRef)), llb.WithCustomName(p.vertexNamef("FetchContainer %s", rawRef)),
) )
@ -517,21 +512,20 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value) error
LogName: p.vertexNamef("load metadata for %s", ref.String()), LogName: p.vertexNamef("load metadata for %s", ref.String()),
}) })
if err != nil { if err != nil {
return err return st, err
} }
for _, env := range image.Config.Env { for _, env := range image.Config.Env {
k, v := parseKeyValue(env) k, v := parseKeyValue(env)
state = state.AddEnv(k, v) st = st.AddEnv(k, v)
} }
if image.Config.WorkingDir != "" { if image.Config.WorkingDir != "" {
state = state.Dir(image.Config.WorkingDir) st = st.Dir(image.Config.WorkingDir)
} }
if image.Config.User != "" { if image.Config.User != "" {
state = state.User(image.Config.User) st = st.User(image.Config.User)
} }
p.fs = p.fs.Set(state) return st, nil
return nil
} }
func parseKeyValue(env string) (string, string) { func parseKeyValue(env string) (string, string) {
@ -544,45 +538,52 @@ func parseKeyValue(env string) (string, string) {
return parts[0], v return parts[0], v
} }
func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
rawRef, err := op.Get("ref").String() rawRef, err := op.Get("ref").String()
if err != nil { if err != nil {
return err return st, err
} }
ref, err := reference.ParseNormalizedNamed(rawRef) ref, err := reference.ParseNormalizedNamed(rawRef)
if err != nil { if err != nil {
return fmt.Errorf("failed to parse ref %s: %w", rawRef, err) return st, fmt.Errorf("failed to parse ref %s: %w", rawRef, err)
} }
// Add the default tag "latest" to a reference if it only has a repo name. // Add the default tag "latest" to a reference if it only has a repo name.
ref = reference.TagNameOnly(ref) ref = reference.TagNameOnly(ref)
_, err = p.fs.Export(ctx, bk.ExportEntry{ pushSt, err := p.result.ToState()
if err != nil {
return st, err
}
_, err = p.s.Export(ctx, pushSt, bk.ExportEntry{
Type: bk.ExporterImage, Type: bk.ExporterImage,
Attrs: map[string]string{ Attrs: map[string]string{
"name": ref.String(), "name": ref.String(),
"push": "true", "push": "true",
}, },
}) })
return err
return st, err
} }
func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
remote, err := op.Get("remote").String() remote, err := op.Get("remote").String()
if err != nil { if err != nil {
return err return st, err
} }
ref, err := op.Get("ref").String() ref, err := op.Get("ref").String()
if err != nil { if err != nil {
return err return st, err
} }
p.fs = p.fs.Set( return llb.Git(
llb.Git(remote, ref, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remote, ref))), remote,
) ref,
return nil llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remote, ref)),
), nil
} }
func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
var ( var (
context = op.Lookup("context") context = op.Lookup("context")
dockerfile = op.Lookup("dockerfile") dockerfile = op.Lookup("dockerfile")
@ -594,7 +595,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
) )
if !context.Exists() && !dockerfile.Exists() { if !context.Exists() && !dockerfile.Exists() {
return errors.New("context or dockerfile required") return st, errors.New("context or dockerfile required")
} }
// docker build context. This can come from another component, so we need to // docker build context. This can come from another component, so we need to
@ -602,11 +603,11 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
if context.Exists() { if context.Exists() {
from := p.Tmp(op.Lookup("context").Path().String()) from := p.Tmp(op.Lookup("context").Path().String())
if err := from.Do(ctx, context); err != nil { if err := from.Do(ctx, context); err != nil {
return err return st, err
} }
contextDef, err = from.FS().Def(ctx) contextDef, err = p.s.Marshal(ctx, from.State())
if err != nil { if err != nil {
return err return st, err
} }
dockerfileDef = contextDef dockerfileDef = contextDef
} }
@ -615,15 +616,15 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
if dockerfile.Exists() { if dockerfile.Exists() {
content, err := dockerfile.String() content, err := dockerfile.String()
if err != nil { if err != nil {
return err return st, err
} }
dockerfileDef, err = p.s.Scratch().Set( dockerfileDef, err = p.s.Marshal(ctx,
llb.Scratch().File( llb.Scratch().File(
llb.Mkfile("/Dockerfile", 0644, []byte(content)), llb.Mkfile("/Dockerfile", 0644, []byte(content)),
), ),
).Def(ctx) )
if err != nil { if err != nil {
return err return st, err
} }
if contextDef == nil { if contextDef == nil {
contextDef = dockerfileDef contextDef = dockerfileDef
@ -642,7 +643,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() { if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() {
filename, err := dockerfilePath.String() filename, err := dockerfilePath.String()
if err != nil { if err != nil {
return err return st, err
} }
req.FrontendOpt["filename"] = filename req.FrontendOpt["filename"] = filename
} }
@ -657,7 +658,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
return nil return nil
}) })
if err != nil { if err != nil {
return err return st, err
} }
} }
@ -671,7 +672,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
return nil return nil
}) })
if err != nil { if err != nil {
return err return st, err
} }
} }
@ -679,13 +680,13 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
p := []string{} p := []string{}
list, err := platforms.List() list, err := platforms.List()
if err != nil { if err != nil {
return err return st, err
} }
for _, platform := range list { for _, platform := range list {
s, err := platform.String() s, err := platform.String()
if err != nil { if err != nil {
return err return st, err
} }
p = append(p, s) p = append(p, s)
} }
@ -700,65 +701,51 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
res, err := p.s.SolveRequest(ctx, req) res, err := p.s.SolveRequest(ctx, req)
if err != nil { if err != nil {
return err return st, err
} }
st, err := res.ToState() return res.ToState()
if err != nil {
return err
}
p.fs = p.fs.Set(st)
return nil
} }
func (p *Pipeline) WriteFile(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) WriteFile(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
content, err := op.Get("content").String() content, err := op.Get("content").String()
if err != nil { if err != nil {
return err return st, err
} }
dest, err := op.Get("dest").String() dest, err := op.Get("dest").String()
if err != nil { if err != nil {
return err return st, err
} }
mode, err := op.Get("mode").Int64() mode, err := op.Get("mode").Int64()
if err != nil { if err != nil {
return err return st, err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.File(
return st.File( llb.Mkfile(dest, fs.FileMode(mode), []byte(content)),
llb.Mkfile(dest, fs.FileMode(mode), []byte(content)), llb.WithCustomName(p.vertexNamef("WriteFile %s", dest)),
llb.WithCustomName(p.vertexNamef("WriteFile %s", dest)), ), nil
)
})
return nil
} }
func (p *Pipeline) Mkdir(ctx context.Context, op *compiler.Value) error { func (p *Pipeline) Mkdir(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
path, err := op.Get("path").String() path, err := op.Get("path").String()
if err != nil { if err != nil {
return err return st, err
} }
dir, err := op.Get("dir").String() dir, err := op.Get("dir").String()
if err != nil { if err != nil {
return err return st, err
} }
mode, err := op.Get("mode").Int64() mode, err := op.Get("mode").Int64()
if err != nil { if err != nil {
return err return st, err
} }
p.fs = p.fs.Change(func(st llb.State) llb.State { return st.Dir(dir).File(
return st.Dir(dir).File( llb.Mkdir(path, fs.FileMode(mode)),
llb.Mkdir(path, fs.FileMode(mode)), llb.WithCustomName(p.vertexNamef("Mkdir %s", path)),
llb.WithCustomName(p.vertexNamef("Mkdir %s", path)), ), nil
)
})
return nil
} }

49
dagger/solver.go
View File

@ -3,7 +3,9 @@ package dagger
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strings"
bk "github.com/moby/buildkit/client" bk "github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb"
@ -16,8 +18,6 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
// Polyfill for buildkit gateway client
// Use instead of bkgw.Client
type Solver struct { type Solver struct {
events chan *bk.SolveStatus events chan *bk.SolveStatus
control *bk.Client control *bk.Client
@ -32,15 +32,13 @@ func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus)
} }
} }
func (s Solver) FS(input llb.State) FS { func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) {
return FS{ // FIXME: do not hardcode the platform
s: s, def, err := st.Marshal(ctx, llb.LinuxAmd64)
input: input, if err != nil {
return nil, err
} }
} return def.ToPB(), nil
func (s Solver) Scratch() FS {
return s.FS(llb.Scratch())
} }
func (s Solver) SessionID() string { func (s Solver) SessionID() string {
@ -78,7 +76,7 @@ func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (bkgw.R
// Solve will block until the state is solved and returns a Reference. // Solve will block until the state is solved and returns a Reference.
func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) { func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error) {
// marshal llb // marshal llb
def, err := st.Marshal(ctx, llb.LinuxAmd64) def, err := s.Marshal(ctx, st)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -96,7 +94,7 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error)
// call solve // call solve
return s.SolveRequest(ctx, bkgw.SolveRequest{ return s.SolveRequest(ctx, bkgw.SolveRequest{
Definition: def.ToPB(), Definition: def,
// makes Solve() to block until LLB graph is solved. otherwise it will // makes Solve() to block until LLB graph is solved. otherwise it will
// return result (that you can for example use for next build) that // return result (that you can for example use for next build) that
@ -110,7 +108,7 @@ func (s Solver) Solve(ctx context.Context, st llb.State) (bkgw.Reference, error)
// within buildkit from the Control API. Ideally the Gateway API should allow to // within buildkit from the Control API. Ideally the Gateway API should allow to
// Export directly. // Export directly.
func (s Solver) Export(ctx context.Context, st llb.State, output bk.ExportEntry) (*bk.SolveResponse, error) { func (s Solver) Export(ctx context.Context, st llb.State, output bk.ExportEntry) (*bk.SolveResponse, error) {
def, err := st.Marshal(ctx, llb.LinuxAmd64) def, err := s.Marshal(ctx, st)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -124,6 +122,8 @@ func (s Solver) Export(ctx context.Context, st llb.State, output bk.ExportEntry)
ch := make(chan *bk.SolveStatus) ch := make(chan *bk.SolveStatus)
// Forward this build session events to the main events channel, for logging
// purposes.
go func() { go func() {
for event := range ch { for event := range ch {
s.events <- event s.events <- event
@ -132,7 +132,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, output bk.ExportEntry)
return s.control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) { return s.control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
return c.Solve(ctx, bkgw.SolveRequest{ return c.Solve(ctx, bkgw.SolveRequest{
Definition: def.ToPB(), Definition: def,
}) })
}, ch) }, ch)
} }
@ -143,7 +143,7 @@ type llbOp struct {
OpMetadata bkpb.OpMetadata OpMetadata bkpb.OpMetadata
} }
func dumpLLB(def *llb.Definition) ([]byte, error) { func dumpLLB(def *bkpb.Definition) ([]byte, error) {
ops := make([]llbOp, 0, len(def.Def)) ops := make([]llbOp, 0, len(def.Def))
for _, dt := range def.Def { for _, dt := range def.Def {
var op bkpb.Op var op bkpb.Op
@ -156,3 +156,22 @@ func dumpLLB(def *llb.Definition) ([]byte, error) {
} }
return json.Marshal(ops) return json.Marshal(ops)
} }
// A helper to remove noise from buildkit error messages.
// FIXME: Obviously a cleaner solution would be nice.
func bkCleanError(err error) error {
noise := []string{
"executor failed running ",
"buildkit-runc did not terminate successfully",
"rpc error: code = Unknown desc = ",
"failed to solve: ",
}
msg := err.Error()
for _, s := range noise {
msg = strings.ReplaceAll(msg, s, "")
}
return errors.New(msg)
}

46
stdlib/stdlib.go
View File

@ -2,48 +2,14 @@ package stdlib
import ( import (
"embed" "embed"
"fmt"
"io/fs"
"path" "path"
"path/filepath"
cueload "cuelang.org/go/cue/load"
) )
// FS contains the filesystem of the stdlib. var (
//go:embed **/*.cue **/*/*.cue // FS contains the filesystem of the stdlib.
var FS embed.FS //go:embed **/*.cue **/*/*.cue
FS embed.FS
const ( PackageName = "dagger.io"
stdlibPackageName = "dagger.io" Path = path.Join("cue.mod", "pkg", PackageName)
) )
func Overlay(prefixPath string) (map[string]cueload.Source, error) {
overlay := map[string]cueload.Source{}
err := fs.WalkDir(FS, ".", func(p string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.Type().IsRegular() {
return nil
}
if filepath.Ext(entry.Name()) != ".cue" {
return nil
}
contents, err := FS.ReadFile(p)
if err != nil {
return fmt.Errorf("%s: %w", p, err)
}
overlayPath := path.Join(prefixPath, "cue.mod", "pkg", stdlibPackageName, p)
overlay[overlayPath] = cueload.FromBytes(contents)
return nil
})
return overlay, err
}