support registry auth

HACK: the way buildkit works, we can only supply an Auth Provider for the entirety of the build session (`dagger up`). Therefore, we start by scanning all auth in the entire Cue tree and supply an auth provider for all of them.

Drawbacks:
- As soon as you add `auth` in a Pipeline for a registry, all other Pipelines have access to the same registry
- You can't use different credentials for the same registry

Fixes #301

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

dagger/client.go

@@ -19,6 +19,7 @@ import (
 	_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
 	"github.com/moby/buildkit/client/llb"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/moby/buildkit/session"
 
 	// docker output
 	"dagger.io/go/pkg/buildkitd"
@@ -101,9 +102,13 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD
 		localdirs[label] = abs
 	}
 
+	// buildkit auth provider (registry)
+	auth := newRegistryAuthProvider()
+
 	// Setup solve options
 	opts := bk.SolveOpt{
 		LocalDirs: localdirs,
+		Session:   []session.Attachable{auth},
 	}
 
 	// Call buildkit solver
@@ -113,13 +118,16 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD
 		Msg("spawning buildkit job")
 
 	resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
-		s := NewSolver(c.c, gw, ch, c.noCache)
+		s := NewSolver(c.c, gw, ch, opts.Session, c.noCache)
 
 		lg.Debug().Msg("loading configuration")
 		if err := deployment.LoadPlan(ctx, s); err != nil {
 			return nil, err
 		}
 
+		lg.Debug().Msg("loading registry credentials from plan")
+		auth.SetCredentials(deployment.RegistryCredentials(ctx))
+
 		// Compute output overlay
 		if fn != nil {
 			if err := fn(ctx, deployment, s); err != nil {

dagger/deployment.go

@@ -12,6 +12,7 @@ import (
 	"dagger.io/go/dagger/compiler"
 	"dagger.io/go/stdlib"
 
+	bkauth "github.com/moby/buildkit/session/auth"
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	otlog "github.com/opentracing/opentracing-go/log"
@@ -113,7 +114,7 @@ func (d *Deployment) LoadPlan(ctx context.Context, s Solver) error {
 	return nil
 }
 
-// Scan all scripts in the deployment for references to local directories (do:"local"),
+// Scan all pipelines in the deployment for references to local directories (do:"local"),
 // and return all referenced directory names.
 // This is used by clients to grant access to local directories when they are referenced
 // by user-specified scripts.
@@ -164,6 +165,75 @@ func (d *Deployment) LocalDirs() map[string]string {
 	return dirs
 }
 
+// Scan all pipelines in the deployment for registry credentials
+func (d *Deployment) RegistryCredentials(ctx context.Context) map[string]*bkauth.CredentialsResponse {
+	credentials := map[string]*bkauth.CredentialsResponse{}
+
+	src, err := compiler.InstanceMerge(d.plan, d.input)
+	if err != nil {
+		panic(err)
+	}
+	flow := cueflow.New(
+		&cueflow.Config{},
+		src.CueInst(),
+		newTaskFunc(src.CueInst(), noOpRunner),
+	)
+
+	authenticatedOps := map[string]struct{}{
+		"push-container":  {},
+		"fetch-container": {},
+		"docker-build":    {},
+	}
+
+	for _, t := range flow.Tasks() {
+		v := compiler.Wrap(t.Value(), src.CueInst())
+		Analyze(
+			func(op *compiler.Value) error {
+				do, err := op.Lookup("do").String()
+				if err != nil {
+					return err
+				}
+				if _, ok := authenticatedOps[do]; !ok {
+					return nil
+				}
+				auth, err := op.Lookup("auth").Fields()
+				if err != nil {
+					return err
+				}
+				for _, a := range auth {
+					host := a.Label
+
+					username, err := a.Value.Lookup("username").String()
+					if err != nil {
+						return err
+					}
+
+					secret, err := a.Value.Lookup("secret").String()
+					if err != nil {
+						return err
+					}
+
+					log.
+						Ctx(ctx).
+						Debug().
+						Str("component", t.Path().String()).
+						Str("host", host).
+						Msg("loading registry credentials")
+
+					credentials[host] = &bkauth.CredentialsResponse{
+						Username: username,
+						Secret:   secret,
+					}
+				}
+				return nil
+			},
+			v.Lookup("#up"),
+		)
+	}
+
+	return credentials
+}
+
 // Up missing values in deployment configuration, and write them to state.
 func (d *Deployment) Up(ctx context.Context, s Solver) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "deployment.Up")

dagger/registryauth.go

@@ -0,0 +1,73 @@
+package dagger
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	bkauth "github.com/moby/buildkit/session/auth"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// registryAuthProvider is a buildkit provider for registry authentication
+// Adapted from: https://github.com/moby/buildkit/blob/master/session/auth/authprovider/authprovider.go
+type registryAuthProvider struct {
+	credentials map[string]*bkauth.CredentialsResponse
+}
+
+func newRegistryAuthProvider() *registryAuthProvider {
+	return &registryAuthProvider{}
+}
+
+func (a *registryAuthProvider) SetCredentials(credentials map[string]*bkauth.CredentialsResponse) {
+	a.credentials = credentials
+}
+
+func (a *registryAuthProvider) Register(server *grpc.Server) {
+	bkauth.RegisterAuthServer(server, a)
+}
+
+func (a *registryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) {
+	reqURL, err := parseAuthHost(req.Host)
+	if err != nil {
+		return nil, err
+	}
+
+	for authHost, auth := range a.credentials {
+		u, err := parseAuthHost(authHost)
+		if err != nil {
+			return nil, err
+		}
+
+		if u.Host == reqURL.Host {
+			return auth, nil
+		}
+	}
+
+	return &bkauth.CredentialsResponse{}, nil
+}
+
+func parseAuthHost(host string) (*url.URL, error) {
+	if host == "registry-1.docker.io" {
+		host = "https://index.docker.io/v1/"
+	}
+
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+		host = "https://" + host
+	}
+	return url.Parse(host)
+}
+
+func (a *registryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) {
+	return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
+}
+
+func (a *registryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) {
+	return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
+}
+
+func (a *registryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) {
+	return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
+}

dagger/solver.go

@@ -13,7 +13,6 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/session/auth/authprovider"
 	bkpb "github.com/moby/buildkit/solver/pb"
 	"github.com/opencontainers/go-digest"
 	"github.com/rs/zerolog/log"
@@ -23,14 +22,16 @@ type Solver struct {
 	events  chan *bk.SolveStatus
 	control *bk.Client
 	gw      bkgw.Client
+	session []session.Attachable
 	noCache bool
 }
 
-func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, noCache bool) Solver {
+func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, session []session.Attachable, noCache bool) Solver {
 	return Solver{
 		events:  events,
 		control: control,
 		gw:      gw,
+		session: session,
 		noCache: noCache,
 	}
 }
@@ -148,9 +149,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
 
 	opts := bk.SolveOpt{
 		Exports: []bk.ExportEntry{output},
-		Session: []session.Attachable{
-			authprovider.NewDockerAuthProvider(log.Ctx(ctx)),
-		},
+		Session: s.session,
 	}
 
 	ch := make(chan *bk.SolveStatus)

go.mod

@@ -29,6 +29,7 @@ require (
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
+	google.golang.org/grpc v1.29.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107172259-749611fa9fcc
 )
 

stdlib/dagger/op/op.cue

@@ -32,7 +32,8 @@ package op
 // FIXME: bring back load (more efficient than copy)
 
 #Load: {
-	do:   "load"
+	do: "load"
+	// FIXME: this should be a `dagger.#Artifact`
 	from: _
 }
 
@@ -48,17 +49,32 @@ package op
 	// `true` means also ignoring the mount cache volumes
 	always?: true | *false
 	dir:     string | *"/"
+	// FIXME: this should be `from: dagger.#Artifact`
 	mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"}
 }
 
+// RegistryCredentials encodes Container Registry credentials
+#RegistryCredentials: {
+	username: string
+	// FIXME: this should `dagger.#Secret`
+	secret: string
+}
+
+// RegistryAuth maps registry hosts to credentials
+#RegistryAuth: {
+	[host=string]: #RegistryCredentials
+}
+
 #FetchContainer: {
-	do:  "fetch-container"
-	ref: string
+	do:   "fetch-container"
+	ref:  string
+	auth: #RegistryAuth
 }
 
 #PushContainer: {
-	do:  "push-container"
-	ref: string
+	do:   "push-container"
+	ref:  string
+	auth: #RegistryAuth
 }
 
 #FetchGit: {
@@ -68,7 +84,8 @@ package op
 }
 
 #Copy: {
-	do:   "copy"
+	do: "copy"
+	// FIXME: this should `dagger.#Artifact`
 	from: _
 	src:  string | *"/"
 	dest: string | *"/"
@@ -77,6 +94,7 @@ package op
 #DockerBuild: {
 	do: "docker-build"
 	// We accept either a context, a Dockerfile or both together
+	// FIXME: this should `dagger.#Artifact`
 	context?:        _
 	dockerfilePath?: string // path to the Dockerfile (defaults to "Dockerfile")
 	dockerfile?:     string
@@ -84,6 +102,10 @@ package op
 	platforms?: [...string]
 	buildArg?: [string]: string
 	label?: [string]:    string
+
+	// credentials for the registry (optional)
+	// used to pull images in `FROM` statements
+	auth: #RegistryAuth
 }
 
 #WriteFile: {

tests/ops.bats

@@ -78,6 +78,11 @@ setup() {
 @test "op.#PushContainer" {
     skip_unless_secrets_available "$TESTDIR"/ops/push-container/inputs.yaml
 
+    # ensure the tests fail without credentials
+    run "$DAGGER" compute "$TESTDIR"/ops/push-container/valid
+    assert_failure
+
+    # check that they succeed with the credentials
     "$DAGGER" compute --input-yaml "$TESTDIR"/ops/push-container/inputs.yaml "$TESTDIR"/ops/push-container
 }
 

tests/ops/push-container/inputs.yaml

@@ -1,13 +1,14 @@
-registry:
-    username: ENC[AES256_GCM,data:8AH6p9WHidanCA==,iv:ezThCQJv+bVBf8SdfSa2HFoP+eu6IZMPl5xvMOGDcps=,tag:mzR7xTKeQNDvkyd2Dm3AKw==,type:str]
-    token: ENC[AES256_GCM,data:68d31b3EfnQJofIt6j+iBCtDyLOBWjFqvVmejyDjIOh8oBXP,iv:PMghC2nd7jqAzrQzm/PW1YdbE0VAbEBkK0/Ri1WwduI=,tag:0JH4WbcJHvgzF4VIK4deBg==,type:str]
+TestAuth:
+    https://index.docker.io/v1/:
+        username: ENC[AES256_GCM,data:cyrR2uKcJBPz9Q==,iv:c6gdvv+OQo/CoFcURoFE5KVdcbsRWFgTC6lOBYqgSKI=,tag:4mXf7SdDzzLOXcuCjNKM3A==,type:str]
+        secret: ENC[AES256_GCM,data:ITq2j0iskLGl3kzLNuqZwjFeY6qhFiZki8+6nfvcCdoQyjwA,iv:wxBQHO2i3usQY6P/Xl7IUNd7FCt92SI4Xzpr0iJf/+Y=,tag:sdT4Vc2bU1WpYBmRx69+NQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
     azure_kv: []
     hc_vault: []
-    lastmodified: '2021-03-18T22:59:59Z'
-    mac: ENC[AES256_GCM,data:3++nHOAJaYFCEuUXim4/gOsG1ZVWt8Ab88qaqHM6jpCA2gLSyADWpB5iQfU9bM7Sq3PgCcWd5+mDHxl5Q8r9fiozrS025OLtsn7qQQQ84WaiFz9Y4Trsbe4EJXNpxYDXjLZEkEtkKs4/Dl+y2Ey3nVyIWKZEX9cPogJ64zfFS9Q=,iv:jvSwxJ8Of2Nfp1ijKItOraDO8aS6aGHQKFY61kF8JS8=,tag:I+AWPIZsPeXU30zxbgq2eQ==,type:str]
+    lastmodified: '2021-04-20T00:38:24Z'
+    mac: ENC[AES256_GCM,data:N7LDE81LW39k9x5Q4JSgcTXiHbXQY9pkJ14g6mIyXd/rtAk8g9nCp2dRSFo75cJDQigh6u4m0AbkvtJ0r1s1Cc87gT+ZXujdb7crEfZLLY23DIEoO1nSgKbh1Snv/uXMDGXG8oCk4pMmUJIc+XtsH0Z/jxuRMVqfyQG7HRm3lVk=,iv:hicEQ0iBDm8hafw6fKCHty5deF0dLr3e2v70LTeYQ1I=,tag:dW3AX5udWxsC3lC2/720RQ==,type:str]
     pgp:
     -   created_at: '2021-03-18T22:59:59Z'
         enc: |

tests/ops/push-container/main.cue

@@ -5,6 +5,8 @@ import (
 	"dagger.io/alpine"
 )
 
+TestAuth: op.#RegistryAuth
+
 TestPushContainer: {
 	// Generate a random number
 	random: {
@@ -13,6 +15,7 @@ TestPushContainer: {
 			op.#Load & {from: alpine.#Image},
 			op.#Exec & {
 				args: ["sh", "-c", "echo -n $RANDOM > /rand"]
+				always: true
 			},
 			op.#Export & {
 				source: "/rand"
@@ -30,6 +33,7 @@ TestPushContainer: {
 			},
 			op.#PushContainer & {
 				"ref": ref
+				auth:  TestAuth
 			},
 		]
 	}
@@ -64,6 +68,7 @@ TestPushContainerMetadata: {
 			op.#Load & {from: alpine.#Image},
 			op.#Exec & {
 				args: ["sh", "-c", "echo -n $RANDOM > /rand"]
+				always: true
 			},
 			op.#Export & {
 				source: "/rand"
@@ -83,6 +88,7 @@ TestPushContainerMetadata: {
 			},
 			op.#PushContainer & {
 				"ref": ref
+				auth:  TestAuth
 			},
 		]
 	}
@@ -113,6 +119,7 @@ TestPushContainerMetadata: {
 			},
 			op.#PushContainer & {
 				"ref": ref
+				auth:  TestAuth
 			},
 		]
 	}