kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

support registry auth

Browse Source 
HACK: the way buildkit works, we can only supply an Auth Provider for the entirety of the build session (`dagger up`). Therefore, we start by scanning all auth in the entire Cue tree and supply an auth provider for all of them.

Drawbacks:
- As soon as you add `auth` in a Pipeline for a registry, all other Pipelines have access to the same registry
- You can't use different credentials for the same registry

Fixes #301

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2021-04-19 18:04:18 -07:00
parent 5e90c1a11e
commit 692bd72095
9 changed files with 204 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dagger/client.go
View File

@ -19,6 +19,7 @@ import (
_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
"github.com/moby/buildkit/client/llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
// docker output
"dagger.io/go/pkg/buildkitd"
@ -101,9 +102,13 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD
localdirs[label] = abs
}
// buildkit auth provider (registry)
auth := newRegistryAuthProvider()
// Setup solve options
opts := bk.SolveOpt{
LocalDirs: localdirs,
Session: []session.Attachable{auth},
}
// Call buildkit solver
@ -113,13 +118,16 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD
Msg("spawning buildkit job")
resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
s := NewSolver(c.c, gw, ch, c.noCache)
s := NewSolver(c.c, gw, ch, opts.Session, c.noCache)
lg.Debug().Msg("loading configuration")
if err := deployment.LoadPlan(ctx, s); err != nil {
return nil, err
}
lg.Debug().Msg("loading registry credentials from plan")
auth.SetCredentials(deployment.RegistryCredentials(ctx))
// Compute output overlay
if fn != nil {
if err := fn(ctx, deployment, s); err != nil {

72
dagger/deployment.go
View File

@ -12,6 +12,7 @@ import (
"dagger.io/go/dagger/compiler"
"dagger.io/go/stdlib"
bkauth "github.com/moby/buildkit/session/auth"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
otlog "github.com/opentracing/opentracing-go/log"
@ -113,7 +114,7 @@ func (d *Deployment) LoadPlan(ctx context.Context, s Solver) error {
return nil
}
// Scan all scripts in the deployment for references to local directories (do:"local"),
// Scan all pipelines in the deployment for references to local directories (do:"local"),
// and return all referenced directory names.
// This is used by clients to grant access to local directories when they are referenced
// by user-specified scripts.
@ -164,6 +165,75 @@ func (d *Deployment) LocalDirs() map[string]string {
return dirs
}
// Scan all pipelines in the deployment for registry credentials
func (d *Deployment) RegistryCredentials(ctx context.Context) map[string]*bkauth.CredentialsResponse {
credentials := map[string]*bkauth.CredentialsResponse{}
src, err := compiler.InstanceMerge(d.plan, d.input)
if err != nil {
panic(err)
}
flow := cueflow.New(
&cueflow.Config{},
src.CueInst(),
newTaskFunc(src.CueInst(), noOpRunner),
)
authenticatedOps := map[string]struct{}{
"push-container": {},
"fetch-container": {},
"docker-build": {},
}
for _, t := range flow.Tasks() {
v := compiler.Wrap(t.Value(), src.CueInst())
Analyze(
func(op *compiler.Value) error {
do, err := op.Lookup("do").String()
if err != nil {
return err
}
if _, ok := authenticatedOps[do]; !ok {
return nil
}
auth, err := op.Lookup("auth").Fields()
if err != nil {
return err
}
for _, a := range auth {
host := a.Label
username, err := a.Value.Lookup("username").String()
if err != nil {
return err
}
secret, err := a.Value.Lookup("secret").String()
if err != nil {
return err
}
log.
Ctx(ctx).
Debug().
Str("component", t.Path().String()).
Str("host", host).
Msg("loading registry credentials")
credentials[host] = &bkauth.CredentialsResponse{
Username: username,
Secret: secret,
}
}
return nil
},
v.Lookup("#up"),
)
}
return credentials
}
// Up missing values in deployment configuration, and write them to state.
func (d *Deployment) Up(ctx context.Context, s Solver) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "deployment.Up")

73
dagger/registryauth.go Normal file
View File

@ -0,0 +1,73 @@
package dagger
import (
"context"
"net/url"
"strings"
bkauth "github.com/moby/buildkit/session/auth"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// registryAuthProvider is a buildkit provider for registry authentication
// Adapted from: https://github.com/moby/buildkit/blob/master/session/auth/authprovider/authprovider.go
type registryAuthProvider struct {
credentials map[string]*bkauth.CredentialsResponse
}
func newRegistryAuthProvider() *registryAuthProvider {
return &registryAuthProvider{}
}
func (a *registryAuthProvider) SetCredentials(credentials map[string]*bkauth.CredentialsResponse) {
a.credentials = credentials
}
func (a *registryAuthProvider) Register(server *grpc.Server) {
bkauth.RegisterAuthServer(server, a)
}
func (a *registryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) {
reqURL, err := parseAuthHost(req.Host)
if err != nil {
return nil, err
}
for authHost, auth := range a.credentials {
u, err := parseAuthHost(authHost)
if err != nil {
return nil, err
}
if u.Host == reqURL.Host {
return auth, nil
}
}
return &bkauth.CredentialsResponse{}, nil
}
func parseAuthHost(host string) (*url.URL, error) {
if host == "registry-1.docker.io" {
host = "https://index.docker.io/v1/"
}
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
host = "https://" + host
}
return url.Parse(host)
}
func (a *registryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}
func (a *registryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}
func (a *registryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) {
return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented")
}

9
dagger/solver.go
View File

@ -13,7 +13,6 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb"
bkgw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth/authprovider"
bkpb "github.com/moby/buildkit/solver/pb"
"github.com/opencontainers/go-digest"
"github.com/rs/zerolog/log"
@ -23,14 +22,16 @@ type Solver struct {
events chan *bk.SolveStatus
control *bk.Client
gw bkgw.Client
session []session.Attachable
noCache bool
}
func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, noCache bool) Solver {
func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, session []session.Attachable, noCache bool) Solver {
return Solver{
events: events,
control: control,
gw: gw,
session: session,
noCache: noCache,
}
}
@ -148,9 +149,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
opts := bk.SolveOpt{
Exports: []bk.ExportEntry{output},
Session: []session.Attachable{
authprovider.NewDockerAuthProvider(log.Ctx(ctx)),
},
Session: s.session,
}
ch := make(chan *bk.SolveStatus)

1
go.mod
View File

@ -29,6 +29,7 @@ require (
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
google.golang.org/grpc v1.29.1 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107172259-749611fa9fcc
)

22
stdlib/dagger/op/op.cue
View File

@ -33,6 +33,7 @@ package op
#Load: {
do: "load"
// FIXME: this should be a `dagger.#Artifact`
from: _
}
@ -48,17 +49,32 @@ package op
// `true` means also ignoring the mount cache volumes
always?: true | *false
dir: string | *"/"
// FIXME: this should be `from: dagger.#Artifact`
mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"}
}
// RegistryCredentials encodes Container Registry credentials
#RegistryCredentials: {
username: string
// FIXME: this should `dagger.#Secret`
secret: string
}
// RegistryAuth maps registry hosts to credentials
#RegistryAuth: {
[host=string]: #RegistryCredentials
}
#FetchContainer: {
do: "fetch-container"
ref: string
auth: #RegistryAuth
}
#PushContainer: {
do: "push-container"
ref: string
auth: #RegistryAuth
}
#FetchGit: {
@ -69,6 +85,7 @@ package op
#Copy: {
do: "copy"
// FIXME: this should `dagger.#Artifact`
from: _
src: string | *"/"
dest: string | *"/"
@ -77,6 +94,7 @@ package op
#DockerBuild: {
do: "docker-build"
// We accept either a context, a Dockerfile or both together
// FIXME: this should `dagger.#Artifact`
context?: _
dockerfilePath?: string // path to the Dockerfile (defaults to "Dockerfile")
dockerfile?: string
@ -84,6 +102,10 @@ package op
platforms?: [...string]
buildArg?: [string]: string
label?: [string]: string
// credentials for the registry (optional)
// used to pull images in `FROM` statements
auth: #RegistryAuth
}
#WriteFile: {

5
tests/ops.bats
View File

@ -78,6 +78,11 @@ setup() {
@test "op.#PushContainer" {
skip_unless_secrets_available "$TESTDIR"/ops/push-container/inputs.yaml
# ensure the tests fail without credentials
run "$DAGGER" compute "$TESTDIR"/ops/push-container/valid
assert_failure
# check that they succeed with the credentials
"$DAGGER" compute --input-yaml "$TESTDIR"/ops/push-container/inputs.yaml "$TESTDIR"/ops/push-container
}

11
tests/ops/push-container/inputs.yaml
View File

@ -1,13 +1,14 @@
registry:
username: ENC[AES256_GCM,data:8AH6p9WHidanCA==,iv:ezThCQJv+bVBf8SdfSa2HFoP+eu6IZMPl5xvMOGDcps=,tag:mzR7xTKeQNDvkyd2Dm3AKw==,type:str]
token: ENC[AES256_GCM,data:68d31b3EfnQJofIt6j+iBCtDyLOBWjFqvVmejyDjIOh8oBXP,iv:PMghC2nd7jqAzrQzm/PW1YdbE0VAbEBkK0/Ri1WwduI=,tag:0JH4WbcJHvgzF4VIK4deBg==,type:str]
TestAuth:
https://index.docker.io/v1/:
username: ENC[AES256_GCM,data:cyrR2uKcJBPz9Q==,iv:c6gdvv+OQo/CoFcURoFE5KVdcbsRWFgTC6lOBYqgSKI=,tag:4mXf7SdDzzLOXcuCjNKM3A==,type:str]
secret: ENC[AES256_GCM,data:ITq2j0iskLGl3kzLNuqZwjFeY6qhFiZki8+6nfvcCdoQyjwA,iv:wxBQHO2i3usQY6P/Xl7IUNd7FCt92SI4Xzpr0iJf/+Y=,tag:sdT4Vc2bU1WpYBmRx69+NQ==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
lastmodified: '2021-03-18T22:59:59Z'
mac: ENC[AES256_GCM,data:3++nHOAJaYFCEuUXim4/gOsG1ZVWt8Ab88qaqHM6jpCA2gLSyADWpB5iQfU9bM7Sq3PgCcWd5+mDHxl5Q8r9fiozrS025OLtsn7qQQQ84WaiFz9Y4Trsbe4EJXNpxYDXjLZEkEtkKs4/Dl+y2Ey3nVyIWKZEX9cPogJ64zfFS9Q=,iv:jvSwxJ8Of2Nfp1ijKItOraDO8aS6aGHQKFY61kF8JS8=,tag:I+AWPIZsPeXU30zxbgq2eQ==,type:str]
lastmodified: '2021-04-20T00:38:24Z'
mac: ENC[AES256_GCM,data:N7LDE81LW39k9x5Q4JSgcTXiHbXQY9pkJ14g6mIyXd/rtAk8g9nCp2dRSFo75cJDQigh6u4m0AbkvtJ0r1s1Cc87gT+ZXujdb7crEfZLLY23DIEoO1nSgKbh1Snv/uXMDGXG8oCk4pMmUJIc+XtsH0Z/jxuRMVqfyQG7HRm3lVk=,iv:hicEQ0iBDm8hafw6fKCHty5deF0dLr3e2v70LTeYQ1I=,tag:dW3AX5udWxsC3lC2/720RQ==,type:str]
pgp:
- created_at: '2021-03-18T22:59:59Z'
enc: |

7
tests/ops/push-container/main.cue
View File

@ -5,6 +5,8 @@ import (
"dagger.io/alpine"
)
TestAuth: op.#RegistryAuth
TestPushContainer: {
// Generate a random number
random: {
@ -13,6 +15,7 @@ TestPushContainer: {
op.#Load & {from: alpine.#Image},
op.#Exec & {
args: ["sh", "-c", "echo -n $RANDOM > /rand"]
always: true
},
op.#Export & {
source: "/rand"
@ -30,6 +33,7 @@ TestPushContainer: {
},
op.#PushContainer & {
"ref": ref
auth: TestAuth
},
]
}
@ -64,6 +68,7 @@ TestPushContainerMetadata: {
op.#Load & {from: alpine.#Image},
op.#Exec & {
args: ["sh", "-c", "echo -n $RANDOM > /rand"]
always: true
},
op.#Export & {
source: "/rand"
@ -83,6 +88,7 @@ TestPushContainerMetadata: {
},
op.#PushContainer & {
"ref": ref
auth: TestAuth
},
]
}
@ -113,6 +119,7 @@ TestPushContainerMetadata: {
},
op.#PushContainer & {
"ref": ref
auth: TestAuth
},
]
}