kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

cleanup: split dagger into sub-packages

Browse Source 
Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2021-05-25 16:30:49 -07:00
parent 93d7bb08e5
commit e13153a284
11 changed files with 114 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/dagger/cmd/common/common.go
View File

@ -3,7 +3,9 @@ package common
import (
"context"
"dagger.io/go/dagger"
"dagger.io/go/dagger/client"
"dagger.io/go/dagger/environment"
"dagger.io/go/dagger/solver"
"dagger.io/go/dagger/state"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
@ -79,14 +81,14 @@ func CurrentEnvironmentState(ctx context.Context, workspace *state.Workspace) *s
}
// Re-compute an environment (equivalent to `dagger up`).
func EnvironmentUp(ctx context.Context, state *state.State, noCache bool) *dagger.Environment {
func EnvironmentUp(ctx context.Context, state *state.State, noCache bool) *environment.Environment {
lg := log.Ctx(ctx)
c, err := dagger.NewClient(ctx, "", noCache)
c, err := client.New(ctx, "", noCache)
if err != nil {
lg.Fatal().Err(err).Msg("unable to create client")
}
result, err := c.Do(ctx, state, func(ctx context.Context, environment *dagger.Environment, s dagger.Solver) error {
result, err := c.Do(ctx, state, func(ctx context.Context, environment *environment.Environment, s solver.Solver) error {
log.Ctx(ctx).Debug().Msg("bringing environment up")
return environment.Up(ctx, s)
})

16
cmd/dagger/cmd/input/list.go
View File

@ -8,8 +8,10 @@ import (
"dagger.io/go/cmd/dagger/cmd/common"
"dagger.io/go/cmd/dagger/logger"
"dagger.io/go/dagger"
"dagger.io/go/dagger/client"
"dagger.io/go/dagger/compiler"
"dagger.io/go/dagger/environment"
"dagger.io/go/dagger/solver"
"dagger.io/go/dagger/state"
"github.com/spf13/cobra"
@ -32,19 +34,19 @@ var listCmd = &cobra.Command{
ctx := lg.WithContext(cmd.Context())
workspace := common.CurrentWorkspace(ctx)
environment := common.CurrentEnvironmentState(ctx, workspace)
st := common.CurrentEnvironmentState(ctx, workspace)
lg = lg.With().
Str("environment", environment.Name).
Str("environment", st.Name).
Logger()
c, err := dagger.NewClient(ctx, "", false)
c, err := client.New(ctx, "", false)
if err != nil {
lg.Fatal().Err(err).Msg("unable to create client")
}
_, err = c.Do(ctx, environment, func(lCtx context.Context, lDeploy *dagger.Environment, lSolver dagger.Solver) error {
inputs := lDeploy.ScanInputs(ctx)
_, err = c.Do(ctx, st, func(ctx context.Context, env *environment.Environment, s solver.Solver) error {
inputs := env.ScanInputs(ctx)
w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
fmt.Fprintln(w, "Input\tType\tValue\tSet by user")
@ -71,7 +73,7 @@ var listCmd = &cobra.Command{
inp.Path(),
getType(inp),
valStr,
isUserSet(environment, inp),
isUserSet(st, inp),
)
}

4
cmd/dagger/cmd/query.go
View File

@ -6,7 +6,7 @@ import (
"cuelang.org/go/cue"
"dagger.io/go/cmd/dagger/cmd/common"
"dagger.io/go/cmd/dagger/logger"
"dagger.io/go/dagger"
"dagger.io/go/dagger/client"
"dagger.io/go/dagger/compiler"
"github.com/spf13/cobra"
@ -42,7 +42,7 @@ var queryCmd = &cobra.Command{
cuePath = cue.ParsePath(args[0])
}
c, err := dagger.NewClient(ctx, "", false)
c, err := client.New(ctx, "", false)
if err != nil {
lg.Fatal().Err(err).Msg("unable to create client")
}

52
dagger/client.go → dagger/client/client.go
View File

@ -1,7 +1,8 @@
package dagger
package client
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
@ -26,6 +27,8 @@ import (
"dagger.io/go/pkg/progressui"
"dagger.io/go/dagger/compiler"
"dagger.io/go/dagger/environment"
"dagger.io/go/dagger/solver"
"dagger.io/go/dagger/state"
)
@ -35,7 +38,7 @@ type Client struct {
noCache bool
}
func NewClient(ctx context.Context, host string, noCache bool) (*Client, error) {
func New(ctx context.Context, host string, noCache bool) (*Client, error) {
if host == "" {
host = os.Getenv("BUILDKIT_HOST")
}
@ -61,14 +64,14 @@ func NewClient(ctx context.Context, host string, noCache bool) (*Client, error)
}, nil
}
type ClientDoFunc func(context.Context, *Environment, Solver) error
type ClientDoFunc func(context.Context, *environment.Environment, solver.Solver) error
// FIXME: return completed *Route, instead of *compiler.Value
func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*Environment, error) {
func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*environment.Environment, error) {
lg := log.Ctx(ctx)
eg, gctx := errgroup.WithContext(ctx)
environment, err := NewEnvironment(state)
environment, err := environment.New(state)
if err != nil {
return nil, err
}
@ -90,11 +93,11 @@ func (c *Client) Do(ctx context.Context, state *state.State, fn ClientDoFunc) (*
return environment, eg.Wait()
}
func (c *Client) buildfn(ctx context.Context, environment *Environment, fn ClientDoFunc, ch chan *bk.SolveStatus) error {
func (c *Client) buildfn(ctx context.Context, env *environment.Environment, fn ClientDoFunc, ch chan *bk.SolveStatus) error {
lg := log.Ctx(ctx)
// Scan local dirs to grant access
localdirs := environment.LocalDirs()
localdirs := env.LocalDirs()
for label, dir := range localdirs {
abs, err := filepath.Abs(dir)
if err != nil {
@ -104,7 +107,7 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien
}
// buildkit auth provider (registry)
auth := newRegistryAuthProvider()
auth := solver.NewRegistryAuthProvider()
// Setup solve options
opts := bk.SolveOpt{
@ -119,16 +122,22 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien
Msg("spawning buildkit job")
resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
s := NewSolver(c.c, gw, ch, auth, c.noCache)
s := solver.New(solver.Opts{
Control: c.c,
Gateway: gw,
Events: ch,
Auth: auth,
NoCache: c.noCache,
})
lg.Debug().Msg("loading configuration")
if err := environment.LoadPlan(ctx, s); err != nil {
if err := env.LoadPlan(ctx, s); err != nil {
return nil, err
}
// Compute output overlay
if fn != nil {
if err := fn(ctx, environment, s); err != nil {
if err := fn(ctx, env, s); err != nil {
return nil, compiler.Err(err)
}
}
@ -139,7 +148,7 @@ func (c *Client) buildfn(ctx context.Context, environment *Environment, fn Clien
span, _ := opentracing.StartSpanFromContext(ctx, "Environment.Export")
defer span.Finish()
computed := environment.Computed().JSON().PrettyString()
computed := env.Computed().JSON().PrettyString()
st := llb.
Scratch().
File(
@ -234,3 +243,22 @@ func (c *Client) logSolveStatus(ctx context.Context, ch chan *bk.SolveStatus) er
},
)
}
// A helper to remove noise from buildkit error messages.
// FIXME: Obviously a cleaner solution would be nice.
func bkCleanError(err error) error {
noise := []string{
"executor failed running ",
"buildkit-runc did not terminate successfully",
"rpc error: code = Unknown desc = ",
"failed to solve: ",
}
msg := err.Error()
for _, s := range noise {
msg = strings.ReplaceAll(msg, s, "")
}
return errors.New(msg)
}

13
dagger/environment.go → dagger/environment/environment.go
View File

@ -1,4 +1,4 @@
package dagger
package environment
import (
"context"
@ -10,6 +10,7 @@ import (
"cuelang.org/go/cue"
cueflow "cuelang.org/go/tools/flow"
"dagger.io/go/dagger/compiler"
"dagger.io/go/dagger/solver"
"dagger.io/go/dagger/state"
"dagger.io/go/stdlib"
@ -32,7 +33,7 @@ type Environment struct {
computed *compiler.Value
}
func NewEnvironment(st *state.State) (*Environment, error) {
func New(st *state.State) (*Environment, error) {
e := &Environment{
state: st,
@ -81,7 +82,7 @@ func (e *Environment) Computed() *compiler.Value {
}
// LoadPlan loads the plan
func (e *Environment) LoadPlan(ctx context.Context, s Solver) error {
func (e *Environment) LoadPlan(ctx context.Context, s solver.Solver) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "environment.LoadPlan")
defer span.Finish()
@ -165,7 +166,7 @@ func (e *Environment) LocalDirs() map[string]string {
}
// Up missing values in environment configuration, and write them to state.
func (e *Environment) Up(ctx context.Context, s Solver) error {
func (e *Environment) Up(ctx context.Context, s solver.Solver) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "environment.Up")
defer span.Finish()
@ -216,7 +217,7 @@ func noOpRunner(t *cueflow.Task) error {
return nil
}
func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc {
func newPipelineRunner(computed *compiler.Value, s solver.Solver) cueflow.RunnerFunc {
return cueflow.RunnerFunc(func(t *cueflow.Task) error {
ctx := t.Context()
lg := log.
@ -294,5 +295,5 @@ func newPipelineRunner(computed *compiler.Value, s Solver) cueflow.RunnerFunc {
}
func (e *Environment) ScanInputs(ctx context.Context) []*compiler.Value {
return ScanInputs(ctx, e.plan)
return scanInputs(ctx, e.plan)
}

4
dagger/environment_test.go → dagger/environment/environment_test.go
View File

@ -1,4 +1,4 @@
package dagger
package environment
import (
"testing"
@ -14,7 +14,7 @@ func TestLocalDirs(t *testing.T) {
}
require.NoError(t, st.SetInput("www.source", state.DirInput("/", []string{})))
environment, err := NewEnvironment(st)
environment, err := New(st)
require.NoError(t, err)
localdirs := environment.LocalDirs()

4
dagger/inputs_scan.go → dagger/environment/inputs_scan.go
View File

@ -1,4 +1,4 @@
package dagger
package environment
import (
"context"
@ -42,7 +42,7 @@ func isReference(val cue.Value) bool {
return isRef(val)
}
func ScanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
func scanInputs(ctx context.Context, value *compiler.Value) []*compiler.Value {
lg := log.Ctx(ctx)
inputs := []*compiler.Value{}

13
dagger/pipeline.go → dagger/environment/pipeline.go
View File

@ -1,4 +1,4 @@
package dagger
package environment
import (
"bytes"
@ -26,6 +26,7 @@ import (
"gopkg.in/yaml.v3"
"dagger.io/go/dagger/compiler"
"dagger.io/go/dagger/solver"
)
const (
@ -36,14 +37,14 @@ const (
type Pipeline struct {
code *compiler.Value
name string
s Solver
s solver.Solver
state llb.State
result bkgw.Reference
image dockerfile2llb.Image
computed *compiler.Value
}
func NewPipeline(code *compiler.Value, s Solver) *Pipeline {
func NewPipeline(code *compiler.Value, s solver.Solver) *Pipeline {
return &Pipeline{
code: code,
name: code.Path().String(),
@ -70,7 +71,7 @@ func (p *Pipeline) Result() (llb.State, error) {
}
func (p *Pipeline) FS() fs.FS {
return NewBuildkitFS(p.result)
return solver.NewBuildkitFS(p.result)
}
func (p *Pipeline) ImageConfig() dockerfile2llb.Image {
@ -641,7 +642,7 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S
return st, err
}
p.s.auth.AddCredentials(target, username, secret)
p.s.AddCredentials(target, username, secret)
log.
Ctx(ctx).
Debug().
@ -862,7 +863,7 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value, st llb.S
return st, err
}
if p.s.noCache {
if p.s.NoCache() {
opts["no-cache"] = ""
}

2
dagger/fs.go → dagger/solver/fs.go
View File

@ -1,4 +1,4 @@
package dagger
package solver
import (
"context"

22
dagger/registryauth.go → dagger/solver/registryauth.go
View File

@ -1,4 +1,4 @@
package dagger
package solver
import (
"context"
@ -12,20 +12,20 @@ import (
"google.golang.org/grpc/status"
)
// registryAuthProvider is a buildkit provider for registry authentication
// RegistryAuthProvider is a buildkit provider for registry authentication
// Adapted from: https://github.com/moby/buildkit/blob/master/session/auth/authprovider/authprovider.go
type registryAuthProvider struct {
type RegistryAuthProvider struct {
credentials map[string]*bkauth.CredentialsResponse
m sync.RWMutex
}
func newRegistryAuthProvider() *registryAuthProvider {
return &registryAuthProvider{
func NewRegistryAuthProvider() *RegistryAuthProvider {
return &RegistryAuthProvider{
credentials: map[string]*bkauth.CredentialsResponse{},
}
}
func (a *registryAuthProvider) AddCredentials(target, username, secret string) {
func (a *RegistryAuthProvider) AddCredentials(target, username, secret string) {
a.m.Lock()
defer a.m.Unlock()
@ -35,11 +35,11 @@ func (a *registryAuthProvider) AddCredentials(target, username, secret string) {
}
}
func (a *registryAuthProvider) Register(server *grpc.Server) {
func (a *RegistryAuthProvider) Register(server *grpc.Server) {
bkauth.RegisterAuthServer(server, a)
}
func (a *registryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) {
func (a *RegistryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) {
reqURL, err := parseAuthHost(req.Host)
if err != nil {
return nil, err
@ -73,14 +73,14 @@ func parseAuthHost(host string) (*url.URL, error) {
return url.Parse(host)
}
func (a *registryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) {
func (a *RegistryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}
func (a *registryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) {
func (a *RegistryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}
func (a *registryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) {
func (a *RegistryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}

72
dagger/solver.go → dagger/solver/solver.go
View File

@ -1,11 +1,9 @@
package dagger
package solver
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
bk "github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
@ -19,20 +17,20 @@ import (
)
type Solver struct {
events chan *bk.SolveStatus
control *bk.Client
gw bkgw.Client
auth *registryAuthProvider
noCache bool
opts Opts
}
func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, auth *registryAuthProvider, noCache bool) Solver {
type Opts struct {
Control *bk.Client
Gateway bkgw.Client
Events chan *bk.SolveStatus
Auth *RegistryAuthProvider
NoCache bool
}
func New(opts Opts) Solver {
return Solver{
events: events,
control: control,
gw: gw,
auth: auth,
noCache: noCache,
opts: opts,
}
}
@ -55,6 +53,14 @@ func invalidateCache(def *llb.Definition) error {
return nil
}
func (s Solver) NoCache() bool {
return s.opts.NoCache
}
func (s Solver) AddCredentials(target, username, secret string) {
s.opts.Auth.AddCredentials(target, username, secret)
}
func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, error) {
// FIXME: do not hardcode the platform
def, err := st.Marshal(ctx, llb.LinuxAmd64)
@ -62,7 +68,7 @@ func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, er
return nil, err
}
if s.noCache {
if s.opts.NoCache {
if err := invalidateCache(def); err != nil {
return nil, err
}
@ -72,7 +78,7 @@ func (s Solver) Marshal(ctx context.Context, st llb.State) (*bkpb.Definition, er
}
func (s Solver) SessionID() string {
return s.gw.BuildOpts().SessionID
return s.opts.Gateway.BuildOpts().SessionID
}
func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.ResolveImageConfigOpt) (dockerfile2llb.Image, error) {
@ -81,7 +87,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res
// Load image metadata and convert to to LLB.
// Inspired by https://github.com/moby/buildkit/blob/master/frontend/dockerfile/dockerfile2llb/convert.go
// FIXME: this needs to handle platform
_, meta, err := s.gw.ResolveImageConfig(ctx, ref, opts)
_, meta, err := s.opts.Gateway.ResolveImageConfig(ctx, ref, opts)
if err != nil {
return image, err
}
@ -94,12 +100,7 @@ func (s Solver) ResolveImageConfig(ctx context.Context, ref string, opts llb.Res
// Solve will block until the state is solved and returns a Reference.
func (s Solver) SolveRequest(ctx context.Context, req bkgw.SolveRequest) (*bkgw.Result, error) {
// call solve
res, err := s.gw.Solve(ctx, req)
if err != nil {
return nil, bkCleanError(err)
}
return res, nil
return s.opts.Gateway.Solve(ctx, req)
}
// Solve will block until the state is solved and returns a Reference.
@ -149,7 +150,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
opts := bk.SolveOpt{
Exports: []bk.ExportEntry{output},
Session: []session.Attachable{s.auth},
Session: []session.Attachable{s.opts.Auth},
}
ch := make(chan *bk.SolveStatus)
@ -158,11 +159,11 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
// purposes.
go func() {
for event := range ch {
s.events <- event
s.opts.Events <- event
}
}()
return s.control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
return s.opts.Control.Build(ctx, opts, "", func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
res, err := c.Solve(ctx, bkgw.SolveRequest{
Definition: def,
})
@ -203,22 +204,3 @@ func dumpLLB(def *bkpb.Definition) ([]byte, error) {
}
return json.Marshal(ops)
}
// A helper to remove noise from buildkit error messages.
// FIXME: Obviously a cleaner solution would be nice.
func bkCleanError(err error) error {
noise := []string{
"executor failed running ",
"buildkit-runc did not terminate successfully",
"rpc error: code = Unknown desc = ",
"failed to solve: ",
}
msg := err.Error()
for _, s := range noise {
msg = strings.ReplaceAll(msg, s, "")
}
return errors.New(msg)
}