kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

prepare the transition to #Plan.context

Browse Source 
This change helps the transition between `dagger input` and `#Plan.context`.

In summary, the codebase now relies on a *context* for execution with mapping to *IDs*.
In the future, *context* will come from a `#Plan.context`.
In the meantime, a bridge converts `dagger input` to a plan context. This allows both *old* and *new* style configurations to co-exist with the same underlying engine.

- Implement `plancontext`. Context holds the execution context for a plan. Currently this includes the platform, local directories, secrets and services (e.g. unix/npipe).
- Contextual data can be registered at any point. In the future, this will be done by `#Plan.context`
- Migrated the `dagger input` codebase to register inputs in a `plancontext`
- Migrated low-level types/operations to the *Context ID* pattern.
  - `dagger.#Stream` now only includes an `id` (instead of `unix` path)
  - `dagger.#Secret` still includes only an ID, but now it's based off `plancontext`
  - `op.#Local` now only includes an `id` (instead of `path`, `include`, `exclude`.

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2021-11-16 16:13:45 -08:00
parent 88312f7f82
commit a61e8dcb62
27 changed files with 401 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
client/client.go
View File

@ -8,7 +8,6 @@ import (
"sync"
"github.com/containerd/containerd/platforms"
"go.opentelemetry.io/otel"
"golang.org/x/sync/errgroup"
"github.com/rs/zerolog/log"
@ -23,13 +22,12 @@ import (
"github.com/moby/buildkit/session"
// docker output
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/util/buildkitd"
"go.dagger.io/dagger/util/progressui"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state"
)
// Client is a dagger client
@ -74,36 +72,31 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) {
}, nil
}
type DoFunc func(context.Context, *environment.Environment, solver.Solver) error
type DoFunc func(context.Context, solver.Solver) error
// FIXME: return completed *Route, instead of *compiler.Value
func (c *Client) Do(ctx context.Context, state *state.State, fn DoFunc) error {
func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error {
lg := log.Ctx(ctx)
eg, gctx := errgroup.WithContext(ctx)
env, err := environment.New(state)
if err != nil {
return err
}
// Spawn print function
events := make(chan *bk.SolveStatus)
eg.Go(func() error {
// Create a background context so that logging will not be cancelled
// with the main context.
dispCtx := lg.WithContext(context.Background())
return c.logSolveStatus(dispCtx, state, events)
return c.logSolveStatus(dispCtx, pctx, events)
})
// Spawn build function
eg.Go(func() error {
return c.buildfn(gctx, state, env, fn, events)
return c.buildfn(gctx, pctx, fn, events)
})
return eg.Wait()
}
func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.Environment, fn DoFunc, ch chan *bk.SolveStatus) error {
func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error {
wg := sync.WaitGroup{}
// Close output channel
@ -115,25 +108,21 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
lg := log.Ctx(ctx)
// Scan local dirs to grant access
localdirs, err := env.LocalDirs()
if err != nil {
return err
}
// buildkit auth provider (registry)
auth := solver.NewRegistryAuthProvider()
// session (secrets & store)
secretsStore := solver.NewSecretsStoreProvider(st)
localdirs := map[string]string{}
for _, dir := range pctx.Directories.List() {
localdirs[dir.Path] = dir.Path
}
// Setup solve options
opts := bk.SolveOpt{
LocalDirs: localdirs,
Session: []session.Attachable{
auth,
secretsStore.Secrets,
solver.NewDockerSocketProvider(),
solver.NewSecretsStoreProvider(pctx),
solver.NewDockerSocketProvider(pctx),
},
CacheExports: c.cfg.CacheExports,
CacheImports: c.cfg.CacheImports,
@ -171,7 +160,6 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
Gateway: gw,
Events: eventsCh,
Auth: auth,
SecretsStore: secretsStore,
NoCache: c.cfg.NoCache,
})
@ -180,28 +168,12 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
// Compute output overlay
if fn != nil {
if err := fn(ctx, env, s); err != nil {
if err := fn(ctx, s); err != nil {
return nil, compiler.Err(err)
}
}
// Export environment to a cue directory
// FIXME: this should be elsewhere
lg.Debug().Msg("exporting environment")
tr := otel.Tracer("client")
_, span := tr.Start(ctx, "environment.Export")
defer span.End()
computed := env.Computed().JSON().PrettyString()
st := llb.
Scratch().
File(
llb.Mkfile("computed.json", 0600, []byte(computed)),
llb.WithCustomName("[internal] serializing computed values"),
)
ref, err := s.Solve(ctx, st, platforms.DefaultSpec())
ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec())
if err != nil {
return nil, err
}
@ -224,7 +196,7 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
return nil
}
func (c *Client) logSolveStatus(ctx context.Context, st *state.State, ch chan *bk.SolveStatus) error {
func (c *Client) logSolveStatus(ctx context.Context, pctx *plancontext.Context, ch chan *bk.SolveStatus) error {
parseName := func(v *bk.Vertex) (string, string) {
// Pattern: `@name@ message`. Minimal length is len("@X@ ")
if len(v.Name) < 2 || !strings.HasPrefix(v.Name, "@") {
@ -241,13 +213,11 @@ func (c *Client) logSolveStatus(ctx context.Context, st *state.State, ch chan *b
}
// Just like sprintf, but redacts secrets automatically
secrets := pctx.Secrets.List()
secureSprintf := func(format string, a ...interface{}) string {
s := fmt.Sprintf(format, a...)
for _, i := range st.Inputs {
if i.Secret == nil {
continue
}
s = strings.ReplaceAll(s, i.Secret.PlainText(), "***")
for _, secret := range secrets {
s = strings.ReplaceAll(s, secret.PlainText, "***")
}
return s
}

9
cmd/dagger/cmd/compute.go
View File

@ -15,6 +15,7 @@ import (
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/environment"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state"
"go.mozilla.org/sops/v3"
@ -43,6 +44,7 @@ var computeCmd = &cobra.Command{
doneCh := common.TrackCommand(ctx, cmd)
st := &state.State{
Context: plancontext.New(),
Name: "FIXME",
Platform: platforms.Format(specs.Platform{OS: "linux", Architecture: "amd64"}),
Path: args[0],
@ -191,7 +193,12 @@ var computeCmd = &cobra.Command{
lg.Fatal().Err(err).Msg("failed to compile inputs")
}
err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
// check that all inputs are set
checkInputs(ctx, env)

7
cmd/dagger/cmd/edit.go
View File

@ -77,8 +77,13 @@ var editCmd = &cobra.Command{
st.Plan = newState.Plan
st.Inputs = newState.Inputs
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
cl := common.NewClient(ctx)
err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
// check for cue errors by scanning all the inputs
_, err := env.ScanInputs(ctx, true)
if err != nil {

33
cmd/dagger/cmd/input/container.go
View File

@ -1,33 +0,0 @@
package input
import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/state"
)
var containerCmd = &cobra.Command{
Use: "container TARGET CONTAINER-IMAGE",
Short: "Add a container image as input artifact",
Args: cobra.ExactArgs(2),
PreRun: func(cmd *cobra.Command, args []string) {
// Fix Viper bug for duplicate flags:
// https://github.com/spf13/viper/issues/233
if err := viper.BindPFlags(cmd.Flags()); err != nil {
panic(err)
}
},
Run: func(cmd *cobra.Command, args []string) {
lg := logger.New()
ctx := lg.WithContext(cmd.Context())
updateEnvironmentInput(ctx, cmd, args[0], state.DockerInput(args[1]))
},
}
func init() {
if err := viper.BindPFlags(containerCmd.Flags()); err != nil {
panic(err)
}
}

9
cmd/dagger/cmd/input/list.go
View File

@ -41,8 +41,13 @@ var listCmd = &cobra.Command{
doneCh := common.TrackProjectCommand(ctx, cmd, project, st)
c := common.NewClient(ctx)
err := c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
cl := common.NewClient(ctx)
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
inputs, err := env.ScanInputs(ctx, false)
if err != nil {
return err

11
cmd/dagger/cmd/input/root.go
View File

@ -25,7 +25,6 @@ func init() {
Cmd.AddCommand(
dirCmd,
gitCmd,
containerCmd,
secretCmd,
textCmd,
jsonCmd,
@ -52,11 +51,15 @@ func updateEnvironmentInput(ctx context.Context, cmd *cobra.Command, target stri
Value: target,
})
cl := common.NewClient(ctx)
st.SetInput(target, input)
err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
cl := common.NewClient(ctx)
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
// the inputs are set, check for cue errors by scanning all the inputs
_, err := env.ScanInputs(ctx, true)
if err != nil {

7
cmd/dagger/cmd/output/list.go
View File

@ -40,8 +40,13 @@ var listCmd = &cobra.Command{
doneCh := common.TrackProjectCommand(ctx, cmd, project, st)
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
cl := common.NewClient(ctx)
err := cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
return ListOutputs(ctx, env, true)
})

7
cmd/dagger/cmd/up.go
View File

@ -61,7 +61,12 @@ var upCmd = &cobra.Command{
cl := common.NewClient(ctx)
err = cl.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
env, err := environment.New(st)
if err != nil {
lg.Fatal().Msg("unable to create environment")
}
err = cl.Do(ctx, env.Context(), func(ctx context.Context, s solver.Solver) error {
// check that all inputs are set
if err := checkInputs(ctx, env); err != nil {
return err

87
environment/environment.go
View File

@ -3,13 +3,11 @@ package environment
import (
"context"
"fmt"
"path/filepath"
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
"github.com/containerd/containerd/platforms"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
"go.dagger.io/dagger/state"
@ -74,59 +72,8 @@ func (e *Environment) Computed() *compiler.Value {
return e.computed
}
// Scan all scripts in the environment for references to local directories (do:"local"),
// and return all referenced directory names.
// This is used by clients to grant access to local directories when they are referenced
// by user-specified scripts.
func (e *Environment) LocalDirs() (map[string]string, error) {
dirs := map[string]string{}
localdirs := func(code *compiler.Value) error {
return Analyze(
func(op *compiler.Value) error {
do, err := op.Lookup("do").String()
if err != nil {
return err
}
if do != "local" {
return nil
}
dir, err := op.Lookup("dir").String()
if err != nil {
return err
}
abs, err := filepath.Abs(dir)
if err != nil {
return err
}
dirs[dir] = abs
return nil
},
code,
)
}
// 1. Scan the environment state
// FIXME: use a common `flow` instance to avoid rescanning the tree.
src := compiler.NewValue()
if err := src.FillPath(cue.MakePath(), e.plan); err != nil {
return nil, err
}
if err := src.FillPath(cue.MakePath(), e.input); err != nil {
return nil, err
}
flow := cueflow.New(
&cueflow.Config{},
src.Cue(),
newTaskFunc(noOpRunner),
)
for _, t := range flow.Tasks() {
if err := localdirs(compiler.Wrap(t.Value())); err != nil {
return nil, err
}
}
return dirs, nil
func (e *Environment) Context() *plancontext.Context {
return e.state.Context
}
// Up missing values in environment configuration, and write them to state.
@ -139,7 +86,7 @@ func (e *Environment) Up(ctx context.Context, s solver.Solver) error {
flow := cueflow.New(
&cueflow.Config{},
e.src.Cue(),
newTaskFunc(newPipelineRunner(e.computed, s, e.state.Platform)),
NewTaskFunc(NewPipelineRunner(e.computed, s, e.state.Context)),
)
if err := flow.Run(ctx); err != nil {
return err
@ -163,7 +110,7 @@ func (e *Environment) Down(ctx context.Context, _ *DownOpts) error {
type QueryOpts struct{}
func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc {
func NewTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc {
return func(flowVal cue.Value) (cueflow.Runner, error) {
v := compiler.Wrap(flowVal)
if !isComponent(v) {
@ -174,11 +121,7 @@ func newTaskFunc(runner cueflow.RunnerFunc) cueflow.TaskFunc {
}
}
func noOpRunner(t *cueflow.Task) error {
return nil
}
func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform string) cueflow.RunnerFunc {
func NewPipelineRunner(computed *compiler.Value, s solver.Solver, pctx *plancontext.Context) cueflow.RunnerFunc {
return cueflow.RunnerFunc(func(t *cueflow.Task) error {
ctx := t.Context()
lg := log.
@ -200,23 +143,7 @@ func newPipelineRunner(computed *compiler.Value, s solver.Solver, platform strin
}
v := compiler.Wrap(t.Value())
var pipelinePlatform specs.Platform
if platform == "" {
pipelinePlatform = specs.Platform{OS: "linux", Architecture: "amd64"}
} else {
p, err := platforms.Parse(platform)
if err != nil {
// Record the error
span.AddEvent("command", trace.WithAttributes(
attribute.String("error", err.Error()),
))
return err
}
pipelinePlatform = p
}
p := NewPipeline(v, s, pipelinePlatform)
p := NewPipeline(v, s, pctx)
err := p.Run(ctx)
if err != nil {
// Record the error

116
environment/pipeline.go
View File

@ -23,11 +23,11 @@ import (
bkgw "github.com/moby/buildkit/frontend/gateway/client"
bkpb "github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog/log"
"gopkg.in/yaml.v3"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
)
@ -46,17 +46,17 @@ type Pipeline struct {
name string
s solver.Solver
state llb.State
platform specs.Platform // Platform constraint
pctx *plancontext.Context
result bkgw.Reference
image dockerfile2llb.Image
computed *compiler.Value
}
func NewPipeline(code *compiler.Value, s solver.Solver, platform specs.Platform) *Pipeline {
func NewPipeline(code *compiler.Value, s solver.Solver, pctx *plancontext.Context) *Pipeline {
return &Pipeline{
code: code,
name: code.Path().String(),
platform: platform,
pctx: pctx,
s: s,
state: llb.Scratch(),
computed: compiler.NewValue(),
@ -233,7 +233,7 @@ func (p *Pipeline) run(ctx context.Context) error {
// so that errors map to the correct cue path.
// FIXME: might as well change FS to make every operation
// synchronous.
p.result, err = p.s.Solve(ctx, p.state, p.platform)
p.result, err = p.s.Solve(ctx, p.state, p.pctx.Platform.Get())
if err != nil {
return err
}
@ -339,7 +339,7 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) (
return st, err
}
// Execute 'from' in a tmp pipeline, and use the resulting fs
from := NewPipeline(op.Lookup("from"), p.s, p.platform)
from := NewPipeline(op.Lookup("from"), p.s, p.pctx)
if err := from.Run(ctx); err != nil {
return st, err
}
@ -361,50 +361,31 @@ func (p *Pipeline) Copy(ctx context.Context, op *compiler.Value, st llb.State) (
}
func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
dir, err := op.Lookup("dir").String()
id, err := op.Lookup("id").String()
if err != nil {
return st, err
}
dir := p.pctx.Directories.Get(plancontext.ContextKey(id))
if dir == nil {
return st, fmt.Errorf("directory %q not found", id)
}
opts := []llb.LocalOption{
llb.WithCustomName(p.vertexNamef("Local %s", dir)),
llb.WithCustomName(p.vertexNamef("Local %s", dir.Path)),
// Without hint, multiple `llb.Local` operations on the
// same path get a different digest.
llb.SessionID(p.s.SessionID()),
llb.SharedKeyHint(dir),
llb.SharedKeyHint(dir.Path),
}
includes, err := op.Lookup("include").List()
if err != nil {
return st, err
}
if len(includes) > 0 {
includePatterns := []string{}
for _, i := range includes {
pattern, err := i.String()
if err != nil {
return st, err
}
includePatterns = append(includePatterns, pattern)
}
opts = append(opts, llb.IncludePatterns(includePatterns))
}
excludes, err := op.Lookup("exclude").List()
if err != nil {
return st, err
if len(dir.Include) > 0 {
opts = append(opts, llb.IncludePatterns(dir.Include))
}
// Excludes .dagger directory by default
excludePatterns := []string{"**/.dagger/"}
if len(excludes) > 0 {
for _, i := range excludes {
pattern, err := i.String()
if err != nil {
return st, err
}
excludePatterns = append(excludePatterns, pattern)
}
if len(dir.Exclude) > 0 {
excludePatterns = dir.Exclude
}
opts = append(opts, llb.ExcludePatterns(excludePatterns))
@ -415,13 +396,13 @@ func (p *Pipeline) Local(ctx context.Context, op *compiler.Value, st llb.State)
return st.File(
llb.Copy(
llb.Local(
dir,
dir.Path,
opts...,
),
"/",
"/",
),
llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir)),
llb.WithCustomName(p.vertexNamef("Local %s [copy]", dir.Path)),
), nil
}
@ -574,43 +555,23 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value)
return nil, fmt.Errorf("invalid stream %q: not a stream", stream.Path().String())
}
// Unix socket
unixValue := stream.Lookup("unix")
if unixValue.Exists() {
unix, err := unixValue.String()
id, err := stream.Lookup("id").String()
if err != nil {
return nil, fmt.Errorf("invalid unix path id: %w", err)
return nil, fmt.Errorf("invalid stream %q: %w", stream.Path().String(), err)
}
return llb.AddSSHSocket(
llb.SSHID(fmt.Sprintf("unix=%s", unix)),
llb.SSHID(id),
llb.SSHSocketTarget(dest),
), nil
}
// Windows named pipe
npipeValue := stream.Lookup("npipe")
if npipeValue.Exists() {
npipe, err := npipeValue.String()
if err != nil {
return nil, fmt.Errorf("invalid npipe path id: %w", err)
}
return llb.AddSSHSocket(
llb.SSHID(fmt.Sprintf("npipe=%s", npipe)),
llb.SSHSocketTarget(dest),
), nil
}
return nil, fmt.Errorf("invalid stream %q: not a valid stream", stream.Path().String())
}
// eg. mount: "/foo": { from: www.source }
if !mnt.Lookup("from").Exists() {
return nil, fmt.Errorf("invalid mount: should have %s structure",
"{from: _, path: string | *\"/\"}")
}
from := NewPipeline(mnt.Lookup("from"), p.s, p.platform)
from := NewPipeline(mnt.Lookup("from"), p.s, p.pctx)
if err := from.Run(ctx); err != nil {
return nil, err
}
@ -736,7 +697,7 @@ func unmarshalAnything(data []byte, fn unmarshaller) (interface{}, error) {
}
// parseStringOrSecret retrieve secret as plain text or retrieve string
func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compiler.Value) (string, error) {
func parseStringOrSecret(pctx *plancontext.Context, v *compiler.Value) (string, error) {
// Check if the value is a string, return as is
if value, err := v.String(); err == nil {
return value, nil
@ -747,16 +708,16 @@ func parseStringOrSecret(ctx context.Context, ss solver.SecretsStore, v *compile
if err != nil {
return "", err
}
secretBytes, err := ss.GetSecret(ctx, id)
if err != nil {
return "", err
secret := pctx.Secrets.Get(plancontext.ContextKey(id))
if secret == nil {
return "", fmt.Errorf("secret %s not found", id)
}
return string(secretBytes), nil
return secret.PlainText, nil
}
func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
// Execute 'from' in a tmp pipeline, and use the resulting fs
from := NewPipeline(op.Lookup("from"), p.s, p.platform)
from := NewPipeline(op.Lookup("from"), p.s, p.pctx)
if err := from.Run(ctx); err != nil {
return st, err
}
@ -774,7 +735,7 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S
// that function
// But currently it's not possible because ECR secret's is a string
// so we need to handle both options (string & secret)
secretValue, err := parseStringOrSecret(ctx, p.s.GetOptions().SecretsStore, op.Lookup("secret"))
secretValue, err := parseStringOrSecret(p.pctx, op.Lookup("secret"))
if err != nil {
return st, err
}
@ -813,9 +774,10 @@ func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st ll
)
// Load image metadata and convert to to LLB.
platform := p.pctx.Platform.Get()
p.image, err = p.s.ResolveImageConfig(ctx, ref.String(), llb.ResolveImageConfigOpt{
LogName: p.vertexNamef("load metadata for %s", ref.String()),
Platform: &p.platform,
Platform: &platform,
})
if err != nil {
return st, err
@ -869,7 +831,7 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb
"name": ref.String(),
"push": "true",
},
}, p.platform)
}, p.pctx.Platform.Get())
if err != nil {
return st, err
@ -928,7 +890,7 @@ func (p *Pipeline) SaveImage(ctx context.Context, op *compiler.Value, st llb.Sta
Output: func(_ map[string]string) (io.WriteCloser, error) {
return pipeW, nil
},
}, p.platform)
}, p.pctx.Platform.Get())
if err != nil {
return st, err
@ -1088,7 +1050,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
// docker build context. This can come from another component, so we need to
// compute it first.
if dockerContext.Exists() {
from := NewPipeline(op.Lookup("context"), p.s, p.platform)
from := NewPipeline(op.Lookup("context"), p.s, p.pctx)
if err := from.Run(ctx); err != nil {
return st, err
}
@ -1118,7 +1080,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
}
}
opts, err := dockerBuildOpts(ctx, op, p.s.GetOptions().SecretsStore)
opts, err := dockerBuildOpts(op, p.pctx)
if err != nil {
return st, err
}
@ -1129,7 +1091,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
// Set platform to configured one if no one is defined
if opts["platform"] == "" {
opts["platform"] = bkplatforms.Format(p.platform)
opts["platform"] = bkplatforms.Format(p.pctx.Platform.Get())
}
req := bkgw.SolveRequest{
@ -1162,7 +1124,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
return applyImageToState(p.image, st), nil
}
func dockerBuildOpts(ctx context.Context, op *compiler.Value, ss solver.SecretsStore) (map[string]string, error) {
func dockerBuildOpts(op *compiler.Value, pctx *plancontext.Context) (map[string]string, error) {
opts := map[string]string{}
if dockerfilePath := op.Lookup("dockerfilePath"); dockerfilePath.Exists() {
@ -1205,7 +1167,7 @@ func dockerBuildOpts(ctx context.Context, op *compiler.Value, ss solver.SecretsS
return nil, err
}
for _, buildArg := range fields {
v, err := parseStringOrSecret(ctx, ss, buildArg.Value)
v, err := parseStringOrSecret(pctx, buildArg.Value)
if err != nil {
return nil, err
}

48
plancontext/context.go Normal file
View File

@ -0,0 +1,48 @@
package plancontext
import (
"crypto/sha256"
"encoding/json"
"fmt"
)
type ContextKey string
// Context holds the execution context for a plan.
//
// Usage:
// ctx := plancontext.New()
// id := ctx.Secrets.Register("mysecret")
// secret := ctx.Secrets.Get(id)
type Context struct {
Platform *platformContext
Directories *directoryContext
Secrets *secretContext
Services *serviceContext
}
func New() *Context {
return &Context{
Platform: &platformContext{
platform: defaultPlatform,
},
Directories: &directoryContext{
store: make(map[ContextKey]*Directory),
},
Secrets: &secretContext{
store: make(map[ContextKey]*Secret),
},
Services: &serviceContext{
store: make(map[ContextKey]*Service),
},
}
}
func hashID(v interface{}) ContextKey {
data, err := json.Marshal(v)
if err != nil {
panic(err)
}
hash := sha256.Sum256(data)
return ContextKey(fmt.Sprintf("%x", hash))
}

42
plancontext/directory.go Normal file
View File

@ -0,0 +1,42 @@
package plancontext
import "sync"
type Directory struct {
Path string
Include []string
Exclude []string
}
type directoryContext struct {
l sync.RWMutex
store map[ContextKey]*Directory
}
func (c *directoryContext) Register(directory *Directory) ContextKey {
c.l.Lock()
defer c.l.Unlock()
id := hashID(directory)
c.store[id] = directory
return id
}
func (c *directoryContext) Get(id ContextKey) *Directory {
c.l.RLock()
defer c.l.RUnlock()
return c.store[id]
}
func (c *directoryContext) List() []*Directory {
c.l.RLock()
defer c.l.RUnlock()
directories := make([]*Directory, 0, len(c.store))
for _, d := range c.store {
directories = append(directories, d)
}
return directories
}

32
plancontext/platform.go Normal file
View File

@ -0,0 +1,32 @@
package plancontext
import (
"github.com/containerd/containerd/platforms"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
var (
// Default platform.
// FIXME: This should be auto detected using buildkit
defaultPlatform = specs.Platform{
OS: "linux",
Architecture: "amd64",
}
)
type platformContext struct {
platform specs.Platform
}
func (c *platformContext) Get() specs.Platform {
return c.platform
}
func (c *platformContext) Set(platform string) error {
p, err := platforms.Parse(platform)
if err != nil {
return err
}
c.platform = p
return nil
}

40
plancontext/secret.go Normal file
View File

@ -0,0 +1,40 @@
package plancontext
import "sync"
type secretContext struct {
l sync.RWMutex
store map[ContextKey]*Secret
}
type Secret struct {
PlainText string
}
func (c *secretContext) Register(secret *Secret) ContextKey {
c.l.Lock()
defer c.l.Unlock()
id := hashID(secret.PlainText)
c.store[id] = secret
return id
}
func (c *secretContext) Get(id ContextKey) *Secret {
c.l.RLock()
defer c.l.RUnlock()
return c.store[id]
}
func (c *secretContext) List() []*Secret {
c.l.RLock()
defer c.l.RUnlock()
secrets := make([]*Secret, 0, len(c.store))
for _, s := range c.store {
secrets = append(secrets, s)
}
return secrets
}

29
plancontext/service.go Normal file
View File

@ -0,0 +1,29 @@
package plancontext
import "sync"
type serviceContext struct {
l sync.RWMutex
store map[ContextKey]*Service
}
type Service struct {
Unix string
Npipe string
}
func (c *serviceContext) Register(service *Service) ContextKey {
c.l.Lock()
defer c.l.Unlock()
id := hashID(service)
c.store[id] = service
return id
}
func (c *serviceContext) Get(id ContextKey) *Service {
c.l.RLock()
defer c.l.RUnlock()
return c.store[id]
}

42
solver/secretsprovider.go
View File

@ -2,55 +2,27 @@ package solver
import (
"context"
"strings"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/state"
"go.dagger.io/dagger/plancontext"
)
type SecretsStore struct {
Secrets session.Attachable
store *inputStore
}
func (s SecretsStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
return s.store.GetSecret(ctx, id)
}
func NewSecretsStoreProvider(st *state.State) SecretsStore {
store := &inputStore{st}
return SecretsStore{
Secrets: secretsprovider.NewSecretProvider(store),
store: store,
}
func NewSecretsStoreProvider(pctx *plancontext.Context) session.Attachable {
return secretsprovider.NewSecretProvider(&inputStore{pctx})
}
type inputStore struct {
st *state.State
pctx *plancontext.Context
}
func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
lg := log.Ctx(ctx)
const secretPrefix = "secret="
if !strings.HasPrefix(id, secretPrefix) {
return nil, secrets.ErrNotFound
}
id = strings.TrimPrefix(id, secretPrefix)
id = strings.Split(id, ";hash=")[0]
input, ok := s.st.Inputs[id]
if !ok {
return nil, secrets.ErrNotFound
}
if input.Secret == nil {
secret := s.pctx.Secrets.Get(plancontext.ContextKey(id))
if secret == nil {
return nil, secrets.ErrNotFound
}
@ -59,5 +31,5 @@ func (s *inputStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
Str("id", id).
Msg("injecting secret")
return []byte(input.Secret.PlainText()), nil
return []byte(secret.PlainText), nil
}

26
solver/socketprovider.go
View File

@ -3,24 +3,20 @@ package solver
import (
"context"
"fmt"
"strings"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/sshforward"
"go.dagger.io/dagger/plancontext"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
const (
unixPrefix = "unix="
npipePrefix = "npipe="
)
type SocketProvider struct {
pctx *plancontext.Context
}
func NewDockerSocketProvider() session.Attachable {
return &SocketProvider{}
func NewDockerSocketProvider(pctx *plancontext.Context) session.Attachable {
return &SocketProvider{pctx}
}
func (sp *SocketProvider) Register(server *grpc.Server) {
@ -28,13 +24,6 @@ func (sp *SocketProvider) Register(server *grpc.Server) {
}
func (sp *SocketProvider) CheckAgent(ctx context.Context, req *sshforward.CheckAgentRequest) (*sshforward.CheckAgentResponse, error) {
id := sshforward.DefaultID
if req.ID != "" {
id = req.ID
}
if !strings.HasPrefix(id, unixPrefix) && !strings.HasPrefix(id, npipePrefix) {
return &sshforward.CheckAgentResponse{}, fmt.Errorf("invalid socket forward key %s", id)
}
return &sshforward.CheckAgentResponse{}, nil
}
@ -47,7 +36,12 @@ func (sp *SocketProvider) ForwardAgent(stream sshforward.SSH_ForwardAgentServer)
id = v[0]
}
conn, err := dialStream(id)
service := sp.pctx.Services.Get(plancontext.ContextKey(id))
if service == nil {
return fmt.Errorf("invalid socket id %q", id)
}
conn, err := dialService(service)
if err != nil {
return fmt.Errorf("failed to connect to %s: %w", id, err)
}

14
solver/socketprovider_unix.go
View File

@ -4,17 +4,17 @@
package solver
import (
"fmt"
"errors"
"net"
"strings"
"time"
"go.dagger.io/dagger/plancontext"
)
func dialStream(id string) (net.Conn, error) {
if !strings.HasPrefix(id, unixPrefix) {
return nil, fmt.Errorf("invalid socket forward key %s", id)
func dialService(service *plancontext.Service) (net.Conn, error) {
if service.Unix == "" {
return nil, errors.New("unsupported socket type")
}
id = strings.TrimPrefix(id, unixPrefix)
return net.DialTimeout("unix", id, time.Second)
return net.DialTimeout("unix", service.Unix, time.Second)
}

13
solver/socketprovider_windows.go
View File

@ -4,20 +4,19 @@
package solver
import (
"fmt"
"errors"
"net"
"strings"
"time"
"github.com/Microsoft/go-winio"
"go.dagger.io/dagger/plancontext"
)
func dialStream(id string) (net.Conn, error) {
if !strings.HasPrefix(id, npipePrefix) {
return nil, fmt.Errorf("invalid socket forward key %s", id)
func dialService(service *plancontext.Service) (net.Conn, error) {
if service.Npipe == "" {
return nil, errors.New("unsupported socket type")
}
id = strings.TrimPrefix(id, npipePrefix)
dur := time.Second
return winio.DialPipe(id, &dur)
return winio.DialPipe(service.Npipe, &dur)
}

7
solver/solver.go
View File

@ -18,6 +18,7 @@ import (
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/plancontext"
)
type Solver struct {
@ -30,8 +31,8 @@ type Opts struct {
Control *bk.Client
Gateway bkgw.Client
Events chan *bk.SolveStatus
Context *plancontext.Context
Auth *RegistryAuthProvider
SecretsStore SecretsStore
NoCache bool
}
@ -194,8 +195,8 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
Exports: []bk.ExportEntry{output},
Session: []session.Attachable{
s.opts.Auth,
s.opts.SecretsStore.Secrets,
NewDockerSocketProvider(),
NewSecretsStoreProvider(s.opts.Context),
NewDockerSocketProvider(s.opts.Context),
},
}

117
state/input.go
View File

@ -1,8 +1,6 @@
package state
import (
"crypto/sha256"
"encoding/json"
"fmt"
"io/ioutil"
"os"
@ -13,6 +11,7 @@ import (
"cuelang.org/go/cue"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
)
// An input is a value or artifact supplied by the user.
@ -31,7 +30,6 @@ import (
type Input struct {
Dir *dirInput `yaml:"dir,omitempty"`
Git *gitInput `yaml:"git,omitempty"`
Docker *dockerInput `yaml:"docker,omitempty"`
Secret *secretInput `yaml:"secret,omitempty"`
Text *textInput `yaml:"text,omitempty"`
JSON *jsonInput `yaml:"json,omitempty"`
@ -41,28 +39,26 @@ type Input struct {
Socket *socketInput `yaml:"socket,omitempty"`
}
func (i Input) Compile(key string, state *State) (*compiler.Value, error) {
func (i Input) Compile(state *State) (*compiler.Value, error) {
switch {
case i.Dir != nil:
return i.Dir.Compile(key, state)
return i.Dir.Compile(state)
case i.Git != nil:
return i.Git.Compile(key, state)
case i.Docker != nil:
return i.Docker.Compile(key, state)
return i.Git.Compile(state)
case i.Text != nil:
return i.Text.Compile(key, state)
return i.Text.Compile(state)
case i.Secret != nil:
return i.Secret.Compile(key, state)
return i.Secret.Compile(state)
case i.JSON != nil:
return i.JSON.Compile(key, state)
return i.JSON.Compile(state)
case i.YAML != nil:
return i.YAML.Compile(key, state)
return i.YAML.Compile(state)
case i.File != nil:
return i.File.Compile(key, state)
return i.File.Compile(state)
case i.Bool != nil:
return i.Bool.Compile(key, state)
return i.Bool.Compile(state)
case i.Socket != nil:
return i.Socket.Compile(key, state)
return i.Socket.Compile(state)
default:
return nil, fmt.Errorf("input has not been set")
}
@ -85,28 +81,7 @@ type dirInput struct {
Exclude []string `yaml:"exclude,omitempty"`
}
func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) {
// FIXME: serialize an intermediate struct, instead of generating cue source
// json.Marshal([]string{}) returns []byte("null"), which wreaks havoc
// in Cue because `null` is not a `[...string]`
includeLLB := []byte("[]")
if len(dir.Include) > 0 {
var err error
includeLLB, err = json.Marshal(dir.Include)
if err != nil {
return nil, err
}
}
excludeLLB := []byte("[]")
if len(dir.Exclude) > 0 {
var err error
excludeLLB, err = json.Marshal(dir.Exclude)
if err != nil {
return nil, err
}
}
func (dir dirInput) Compile(state *State) (*compiler.Value, error) {
p := dir.Path
if !filepath.IsAbs(p) {
p = filepath.Clean(path.Join(state.Project, dir.Path))
@ -119,16 +94,15 @@ func (dir dirInput) Compile(_ string, state *State) (*compiler.Value, error) {
return nil, fmt.Errorf("%q dir doesn't exist", dir.Path)
}
dirPath, err := json.Marshal(p)
if err != nil {
return nil, err
}
id := state.Context.Directories.Register(&plancontext.Directory{
Path: p,
Include: dir.Include,
Exclude: dir.Exclude,
})
llb := fmt.Sprintf(
`#up: [{do:"local",dir:%s, include:%s, exclude:%s}]`,
dirPath,
includeLLB,
excludeLLB,
`#up: [{do:"local", id: "%s"}]`,
id,
)
return compiler.Compile("", llb)
}
@ -150,7 +124,7 @@ func GitInput(remote, ref, dir string) Input {
}
}
func (git gitInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (git gitInput) Compile(_ *State) (*compiler.Value, error) {
ref := "HEAD"
if git.Ref != "" {
ref = git.Ref
@ -169,23 +143,6 @@ func (git gitInput) Compile(_ string, _ *State) (*compiler.Value, error) {
))
}
// An input artifact loaded from a docker container
func DockerInput(ref string) Input {
return Input{
Docker: &dockerInput{
Ref: ref,
},
}
}
type dockerInput struct {
Ref string `yaml:"ref,omitempty"`
}
func (i dockerInput) Compile(_ string, _ *State) (*compiler.Value, error) {
panic("NOT IMPLEMENTED")
}
// An input value encoded as text
func TextInput(data string) Input {
i := textInput(data)
@ -196,7 +153,7 @@ func TextInput(data string) Input {
type textInput string
func (i textInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (i textInput) Compile(_ *State) (*compiler.Value, error) {
return compiler.Compile("", fmt.Sprintf("%q", i))
}
@ -210,11 +167,11 @@ func SecretInput(data string) Input {
type secretInput string
func (i secretInput) Compile(key string, _ *State) (*compiler.Value, error) {
hash := sha256.New()
hash.Write([]byte(key))
checksum := hash.Sum([]byte(i.PlainText()))
secretValue := fmt.Sprintf(`{id:"secret=%s;hash=%x"}`, key, checksum)
func (i secretInput) Compile(st *State) (*compiler.Value, error) {
id := st.Context.Secrets.Register(&plancontext.Secret{
PlainText: i.PlainText(),
})
secretValue := fmt.Sprintf(`{id: %q}`, id)
return compiler.Compile("", secretValue)
}
@ -232,7 +189,7 @@ func BoolInput(data string) Input {
type boolInput string
func (i boolInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (i boolInput) Compile(_ *State) (*compiler.Value, error) {
s := map[boolInput]struct{}{
"true": {},
"false": {},
@ -253,7 +210,7 @@ func JSONInput(data string) Input {
type jsonInput string
func (i jsonInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (i jsonInput) Compile(_ *State) (*compiler.Value, error) {
return compiler.DecodeJSON("", []byte(i))
}
@ -267,7 +224,7 @@ func YAMLInput(data string) Input {
type yamlInput string
func (i yamlInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (i yamlInput) Compile(_ *State) (*compiler.Value, error) {
return compiler.DecodeYAML("", []byte(i))
}
@ -283,7 +240,7 @@ type fileInput struct {
Path string `yaml:"path,omitempty"`
}
func (i fileInput) Compile(_ string, _ *State) (*compiler.Value, error) {
func (i fileInput) Compile(_ *State) (*compiler.Value, error) {
data, err := ioutil.ReadFile(i.Path)
if err != nil {
return nil, err
@ -316,11 +273,11 @@ type socketInput struct {
Npipe string `json:"npipe,omitempty" yaml:"npipe,omitempty"`
}
func (i socketInput) Compile(_ string, _ *State) (*compiler.Value, error) {
socketValue, err := json.Marshal(i)
if err != nil {
return nil, err
}
return compiler.Compile("", string(socketValue))
func (i socketInput) Compile(st *State) (*compiler.Value, error) {
id := st.Context.Services.Register(&plancontext.Service{
Unix: i.Unix,
Npipe: i.Npipe,
})
socketValue := fmt.Sprintf(`{id: %q}`, id)
return compiler.Compile("", socketValue)
}

9
state/project.go
View File

@ -12,6 +12,7 @@ import (
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/keychain"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/stdlib"
"gopkg.in/yaml.v3"
)
@ -168,6 +169,12 @@ func (w *Project) Get(ctx context.Context, name string) (*State, error) {
if err := yaml.Unmarshal(manifest, &st); err != nil {
return nil, err
}
st.Context = plancontext.New()
if platform := st.Platform; platform != "" {
if err := st.Context.Platform.Set(platform); err != nil {
return nil, err
}
}
st.Path = envPath
// FIXME: Backward compat: Support for old-style `.dagger/env/<name>/plan`
if st.Plan.Module == "" {
@ -258,6 +265,8 @@ func (w *Project) Create(ctx context.Context, name string, plan Plan, platform s
manifestPath := path.Join(envPath, manifestFile)
st := &State{
Context: plancontext.New(),
Path: envPath,
Project: w.Path,
Plan: Plan{

7
state/state.go
View File

@ -6,10 +6,15 @@ import (
"cuelang.org/go/cue"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
)
// Contents of an environment serialized to a file
type State struct {
// Plan Context.
// FIXME: this is used as a bridge and is temporary.
Context *plancontext.Context `yaml:"-"`
// State path
Path string `yaml:"-"`
@ -67,7 +72,7 @@ func (s *State) CompileInputs() (*compiler.Value, error) {
// Prepare inputs
for key, input := range s.Inputs {
i, err := input.Compile(key, s)
i, err := input.Compile(s)
if err != nil {
return nil, err
}

8
stdlib/dagger/dagger.cue
View File

@ -18,13 +18,7 @@ import (
#Stream: {
@dagger(stream)
{
// Unix Socket
unix: string
} | {
// Windows Named Pipe
npipe: string
}
id: string
}
// Secret value

4
stdlib/dagger/op/op.cue
View File

@ -21,9 +21,7 @@ package op
#Local: {
do: "local"
dir: string
include: [...string]
exclude: [...string]
id: string
}
// FIXME: bring back load (more efficient than copy)

3
stdlib/docker/docker.cue
View File

@ -160,6 +160,9 @@ import (
string
#up: [
// HACK: force a dependency with `load`
op.#Load & {from: load},
op.#Load & {from: save},
op.#Export & {

1
tests/core.bats
View File

@ -145,7 +145,6 @@ setup() {
# Make sure the secret doesn't show in dagger query
run "$DAGGER" query mySecret.id -f text
assert_success
assert_output --partial "secret=mySecret;hash="
}
@test "core: stream" {