From c28199c76e12949e7c15153a680fdf2978abdbf5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 23 Apr 2021 17:29:28 -0700 Subject: [PATCH 1/8] stdlib/ecr / jamstack: ported to new syntax Signed-off-by: Sam Alba --- examples/jamstack/ecr_image.cue | 12 ++++++++---- stdlib/aws/ecr/ecr.cue | 15 +++------------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/examples/jamstack/ecr_image.cue b/examples/jamstack/ecr_image.cue index 01ce4cba..4cc58f7a 100644 --- a/examples/jamstack/ecr_image.cue +++ b/examples/jamstack/ecr_image.cue @@ -17,12 +17,9 @@ import ( awsConfig: aws.#Config buildArgs: [string]: string - pushTarget: "\(repository):\(tag)" - // Use these credentials to push ecrCreds: ecr.#Credentials & { config: awsConfig - target: pushTarget } ref: { @@ -37,9 +34,16 @@ import ( } buildArg: buildArgs }, + // Login to Registry + op.#DockerLogin & { + target: repository + username: ecrCreds.username + secret: ecrCreds.secret + }, // Push the image to the registry op.#PushContainer & { - ref: pushTarget + ref: "\(repository):\(tag)" + auth: ecrCreds.auth }, op.#Export & { source: "/dagger/image_ref" diff --git a/stdlib/aws/ecr/ecr.cue b/stdlib/aws/ecr/ecr.cue index 75b0c050..75b7de36 100644 --- a/stdlib/aws/ecr/ecr.cue +++ b/stdlib/aws/ecr/ecr.cue @@ -7,30 +7,21 @@ import ( // Credentials retriever for ECR #Credentials: { - // AWS Config config: aws.#Config - // Target is the ECR image - target: string - out: dagger.#Secret // ECR credentials - credentials: dagger.#RegistryCredentials & { - username: "AWS" - secret: out - } + username: "AWS" + secret: out aws.#Script & { + always: true "config": config export: "/out" code: """ aws ecr get-login-password > /out """ } - - // Authentication for ECR Registries - auth: dagger.#RegistryAuth - auth: "\(target)": credentials } From 45ecc324236d6c9b6288dddeff3f22db12350162 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:39:19 -0700 Subject: [PATCH 2/8] implemented docker-login op to add custom registry credentials to the buildkit from a pipeline Signed-off-by: Sam Alba --- dagger/client.go | 4 +- dagger/pipeline.go | 28 ++++++++++++++ dagger/registryauth.go | 86 ++++++++++++++++++++++++++++++++++++++++++ dagger/solver.go | 9 ++--- 4 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 dagger/registryauth.go diff --git a/dagger/client.go b/dagger/client.go index 3635301a..5b3674d3 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -113,7 +113,9 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD Msg("spawning buildkit job") resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { - s := NewSolver(c.c, gw, ch, c.noCache) + // buildkit auth provider (registry) + auth := newRegistryAuthProvider() + s := NewSolver(c.c, gw, ch, auth, c.noCache) lg.Debug().Msg("loading configuration") if err := deployment.LoadPlan(ctx, s); err != nil { diff --git a/dagger/pipeline.go b/dagger/pipeline.go index f0ca2999..12e178ef 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -197,6 +197,8 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value, st llb.State) ( return p.Exec(ctx, op, st) case "export": return p.Export(ctx, op, st) + case "docker-login": + return p.DockerLogin(ctx, op, st) case "fetch-container": return p.FetchContainer(ctx, op, st) case "push-container": @@ -559,6 +561,32 @@ func (p *Pipeline) Load(ctx context.Context, op *compiler.Value, st llb.State) ( return from.State(), nil } +func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { + username, err := op.Lookup("username").String() + if err != nil { + return st, err + } + + secret, err := op.Lookup("secret").String() + if err != nil { + return st, err + } + + target, err := op.Lookup("target").String() + if err != nil { + return st, err + } + + p.s.auth.AddCredentials(target, username, secret) + log. + Ctx(ctx). + Debug(). + Str("target", target). + Msg("docker login to registry") + + return st, nil +} + func (p *Pipeline) FetchContainer(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) { rawRef, err := op.Lookup("ref").String() if err != nil { diff --git a/dagger/registryauth.go b/dagger/registryauth.go new file mode 100644 index 00000000..5acd618a --- /dev/null +++ b/dagger/registryauth.go @@ -0,0 +1,86 @@ +package dagger + +import ( + "context" + "net/url" + "strings" + "sync" + + bkauth "github.com/moby/buildkit/session/auth" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// registryAuthProvider is a buildkit provider for registry authentication +// Adapted from: https://github.com/moby/buildkit/blob/master/session/auth/authprovider/authprovider.go +type registryAuthProvider struct { + credentials map[string]*bkauth.CredentialsResponse + m sync.RWMutex +} + +func newRegistryAuthProvider() *registryAuthProvider { + return ®istryAuthProvider{ + credentials: map[string]*bkauth.CredentialsResponse{}, + } +} + +func (a *registryAuthProvider) AddCredentials(target, username, secret string) { + a.m.Lock() + defer a.m.Unlock() + + a.credentials[target] = &bkauth.CredentialsResponse{ + Username: username, + Secret: secret, + } +} + +func (a *registryAuthProvider) Register(server *grpc.Server) { + bkauth.RegisterAuthServer(server, a) +} + +func (a *registryAuthProvider) Credentials(ctx context.Context, req *bkauth.CredentialsRequest) (*bkauth.CredentialsResponse, error) { + reqURL, err := parseAuthHost(req.Host) + if err != nil { + return nil, err + } + + a.m.RLock() + defer a.m.RUnlock() + + for authHost, auth := range a.credentials { + u, err := parseAuthHost(authHost) + if err != nil { + return nil, err + } + + if u.Host == reqURL.Host { + return auth, nil + } + } + + return &bkauth.CredentialsResponse{}, nil +} + +func parseAuthHost(host string) (*url.URL, error) { + if host == "registry-1.docker.io" { + host = "https://index.docker.io/v1/" + } + + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + host = "https://" + host + } + return url.Parse(host) +} + +func (a *registryAuthProvider) FetchToken(ctx context.Context, req *bkauth.FetchTokenRequest) (rr *bkauth.FetchTokenResponse, err error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") +} + +func (a *registryAuthProvider) GetTokenAuthority(ctx context.Context, req *bkauth.GetTokenAuthorityRequest) (*bkauth.GetTokenAuthorityResponse, error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") +} + +func (a *registryAuthProvider) VerifyTokenAuthority(ctx context.Context, req *bkauth.VerifyTokenAuthorityRequest) (*bkauth.VerifyTokenAuthorityResponse, error) { + return nil, status.Errorf(codes.Unavailable, "client side tokens not implemented") +} diff --git a/dagger/solver.go b/dagger/solver.go index 3a78c0c1..e057155b 100644 --- a/dagger/solver.go +++ b/dagger/solver.go @@ -13,7 +13,6 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth/authprovider" bkpb "github.com/moby/buildkit/solver/pb" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" @@ -23,14 +22,16 @@ type Solver struct { events chan *bk.SolveStatus control *bk.Client gw bkgw.Client + auth *registryAuthProvider noCache bool } -func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, noCache bool) Solver { +func NewSolver(control *bk.Client, gw bkgw.Client, events chan *bk.SolveStatus, auth *registryAuthProvider, noCache bool) Solver { return Solver{ events: events, control: control, gw: gw, + auth: auth, noCache: noCache, } } @@ -148,9 +149,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im opts := bk.SolveOpt{ Exports: []bk.ExportEntry{output}, - Session: []session.Attachable{ - authprovider.NewDockerAuthProvider(log.Ctx(ctx)), - }, + Session: []session.Attachable{s.auth}, } ch := make(chan *bk.SolveStatus) From 4df8b3e0870c61635dba33e02a376317c8ca5d03 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:41:04 -0700 Subject: [PATCH 3/8] stdlib/dagger/op: Added #DockerLogin Signed-off-by: Sam Alba --- stdlib/dagger/op/op.cue | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/stdlib/dagger/op/op.cue b/stdlib/dagger/op/op.cue index 92ec4c53..cbf09096 100644 --- a/stdlib/dagger/op/op.cue +++ b/stdlib/dagger/op/op.cue @@ -51,6 +51,14 @@ package op mount: [string]: "tmpfs" | "cache" | {from: _, path: string | *"/"} } +#DockerLogin: { + do: "docker-login" + target: string | *"https://index.docker.io/v1/" + username: string + // FIXME: should be a #Secret (circular import) + secret: string | bytes +} + #FetchContainer: { do: "fetch-container" ref: string From ddd093f44dd85775da81d62183ce32fd7d5b8959 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:42:13 -0700 Subject: [PATCH 4/8] examples/jamstack: updated example Signed-off-by: Sam Alba --- examples/jamstack/ecr_image.cue | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/jamstack/ecr_image.cue b/examples/jamstack/ecr_image.cue index 4cc58f7a..01631857 100644 --- a/examples/jamstack/ecr_image.cue +++ b/examples/jamstack/ecr_image.cue @@ -43,7 +43,6 @@ import ( // Push the image to the registry op.#PushContainer & { ref: "\(repository):\(tag)" - auth: ecrCreds.auth }, op.#Export & { source: "/dagger/image_ref" From ea43129a1f1c5b72aa0044ae0d3b4a03db9ec761 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:42:36 -0700 Subject: [PATCH 5/8] go mod Signed-off-by: Sam Alba --- go.mod | 1 + go.sum | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 65402da6..1c498e09 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 + google.golang.org/grpc v1.29.1 gopkg.in/yaml.v3 v3.0.0-20210107172259-749611fa9fcc ) diff --git a/go.sum b/go.sum index d15b8536..8000577e 100644 --- a/go.sum +++ b/go.sum @@ -305,7 +305,6 @@ github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r github.com/docker/docker v17.12.0-ce-rc1.0.20200730172259-9f28837c1d93+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible h1:J2OhsbfqoBRRT048iD/tqXBvEQWQATQ8vew6LqQmDSU= github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= From aa74b8fb7b0eb7f41fad9183e220b9e37b91c6df Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:42:48 -0700 Subject: [PATCH 6/8] added tests for push-container with login Signed-off-by: Sam Alba --- tests/ops.bats | 5 +++++ tests/ops/push-container/main.cue | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/ops.bats b/tests/ops.bats index 39e23514..0bc7135e 100644 --- a/tests/ops.bats +++ b/tests/ops.bats @@ -84,6 +84,11 @@ setup() { @test "op.#PushContainer" { skip_unless_secrets_available "$TESTDIR"/ops/push-container/inputs.yaml + # ensure the tests fail without credentials + run "$DAGGER" compute "$TESTDIR"/ops/push-container/valid + assert_failure + + # check that they succeed with the credentials run "$DAGGER" compute --input-yaml "$TESTDIR"/ops/push-container/inputs.yaml "$TESTDIR"/ops/push-container assert_success } diff --git a/tests/ops/push-container/main.cue b/tests/ops/push-container/main.cue index e3930270..7129990f 100644 --- a/tests/ops/push-container/main.cue +++ b/tests/ops/push-container/main.cue @@ -1,10 +1,16 @@ package testing import ( + "dagger.io/dagger" "dagger.io/dagger/op" "dagger.io/alpine" ) +registry: { + username: string + secret: dagger.#Secret +} + TestPushContainer: { // Generate a random number random: { @@ -24,6 +30,9 @@ TestPushContainer: { push: { ref: "daggerio/ci-test:\(random)" #up: [ + op.#DockerLogin & { + registry + }, op.#WriteFile & { content: random dest: "/rand" From df4f4b293c7c7d89e2935fddb0db985ecded1b77 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 17:45:14 -0700 Subject: [PATCH 7/8] cue fmt Signed-off-by: Sam Alba --- examples/jamstack/ecr_image.cue | 2 +- tests/ops/push-container/main.cue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/jamstack/ecr_image.cue b/examples/jamstack/ecr_image.cue index 01631857..a98c5544 100644 --- a/examples/jamstack/ecr_image.cue +++ b/examples/jamstack/ecr_image.cue @@ -42,7 +42,7 @@ import ( }, // Push the image to the registry op.#PushContainer & { - ref: "\(repository):\(tag)" + ref: "\(repository):\(tag)" }, op.#Export & { source: "/dagger/image_ref" diff --git a/tests/ops/push-container/main.cue b/tests/ops/push-container/main.cue index 7129990f..80db13f1 100644 --- a/tests/ops/push-container/main.cue +++ b/tests/ops/push-container/main.cue @@ -8,7 +8,7 @@ import ( registry: { username: string - secret: dagger.#Secret + secret: dagger.#Secret } TestPushContainer: { From 520030d4ee64ac7caeb76c3e17eae952bba7c122 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 26 Apr 2021 18:00:03 -0700 Subject: [PATCH 8/8] fixed tests creds + added authprovider to client solver Signed-off-by: Sam Alba --- dagger/client.go | 7 +++++-- tests/ops/push-container/inputs.yaml | 15 ++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dagger/client.go b/dagger/client.go index 5b3674d3..e78547f3 100644 --- a/dagger/client.go +++ b/dagger/client.go @@ -19,6 +19,7 @@ import ( _ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver "github.com/moby/buildkit/client/llb" bkgw "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/session" // docker output "dagger.io/go/pkg/buildkitd" @@ -101,9 +102,13 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD localdirs[label] = abs } + // buildkit auth provider (registry) + auth := newRegistryAuthProvider() + // Setup solve options opts := bk.SolveOpt{ LocalDirs: localdirs, + Session: []session.Attachable{auth}, } // Call buildkit solver @@ -113,8 +118,6 @@ func (c *Client) buildfn(ctx context.Context, deployment *Deployment, fn ClientD Msg("spawning buildkit job") resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) { - // buildkit auth provider (registry) - auth := newRegistryAuthProvider() s := NewSolver(c.c, gw, ch, auth, c.noCache) lg.Debug().Msg("loading configuration") diff --git a/tests/ops/push-container/inputs.yaml b/tests/ops/push-container/inputs.yaml index c3c9fb1e..f054b2cf 100644 --- a/tests/ops/push-container/inputs.yaml +++ b/tests/ops/push-container/inputs.yaml @@ -1,16 +1,17 @@ registry: username: ENC[AES256_GCM,data:8AH6p9WHidanCA==,iv:ezThCQJv+bVBf8SdfSa2HFoP+eu6IZMPl5xvMOGDcps=,tag:mzR7xTKeQNDvkyd2Dm3AKw==,type:str] - token: ENC[AES256_GCM,data:68d31b3EfnQJofIt6j+iBCtDyLOBWjFqvVmejyDjIOh8oBXP,iv:PMghC2nd7jqAzrQzm/PW1YdbE0VAbEBkK0/Ri1WwduI=,tag:0JH4WbcJHvgzF4VIK4deBg==,type:str] + secret: ENC[AES256_GCM,data:GtuaBAhFBw2JFaeuOm6mUr3m1j5fvCJjcWAzjsdU2xASFxwO,iv:YAXcRzBoemmef5PBdAOBa5acNPo4BoKH7Ngud/CWYfA=,tag:MCCUCOSutjRCI92raYrxdg==,type:str] sops: kms: [] gcp_kms: [] azure_kv: [] hc_vault: [] - lastmodified: '2021-03-18T22:59:59Z' - mac: ENC[AES256_GCM,data:3++nHOAJaYFCEuUXim4/gOsG1ZVWt8Ab88qaqHM6jpCA2gLSyADWpB5iQfU9bM7Sq3PgCcWd5+mDHxl5Q8r9fiozrS025OLtsn7qQQQ84WaiFz9Y4Trsbe4EJXNpxYDXjLZEkEtkKs4/Dl+y2Ey3nVyIWKZEX9cPogJ64zfFS9Q=,iv:jvSwxJ8Of2Nfp1ijKItOraDO8aS6aGHQKFY61kF8JS8=,tag:I+AWPIZsPeXU30zxbgq2eQ==,type:str] + age: [] + lastmodified: "2021-04-27T00:59:33Z" + mac: ENC[AES256_GCM,data:qk+oo4m5OpfuQ+R3pZUuvn+gqAk15OAJzOULrlYqt1FIDRk/Q5ah5QpIbVxeP1EDVyuY/V/E0ZngRlSV7Dyx6Cp/moMd8AFBHNgnTB+Lq+NmZ9HR1QMOxpbMpJmUGn7MqQ1Ys4wy0p2q2Y2+TuUpKwmRGJbGVYEVmqvV5OT3jhc=,iv:QsUFa2GVzy6iqqLXRz8HascQZPIIzKBhxHdlabov02k=,tag:7lk63FeXsOlTCgfmWd7zrg==,type:str] pgp: - - created_at: '2021-03-18T22:59:59Z' - enc: | + - created_at: "2021-03-18T22:59:59Z" + enc: | -----BEGIN PGP MESSAGE----- hQIMAzqVY590vudzAQ//etnfnpfCo9rAkctR+Fwg/7VdVL3Rov+6gnyjUnoN1BS1 @@ -28,6 +29,6 @@ sops: Xd3gV3smg5xZ7/rfvzKTzJ1a5yH6D3xI05UtnUWdqojONcXS9NS+P7RArngJwSs= =m0OS -----END PGP MESSAGE----- - fp: 6CB37404020B5F0A0B41B5BB225EBAB0B936AC65 + fp: 6CB37404020B5F0A0B41B5BB225EBAB0B936AC65 unencrypted_suffix: _unencrypted - version: 3.6.1 + version: 3.7.1