Merge pull request #929 from TomChv/feat/docker-secret

op.#PushContainer secret management

client/client.go

@@ -128,15 +128,15 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
 	// buildkit auth provider (registry)
 	auth := solver.NewRegistryAuthProvider()
 
-	// secrets
+	// session (secrets & store)
-	secrets := solver.NewSecretsProvider(st)
+	secretsStore := solver.NewSecretsStoreProvider(st)
 
 	// Setup solve options
 	opts := bk.SolveOpt{
 		LocalDirs: localdirs,
 		Session: []session.Attachable{
 			auth,
-			secrets,
+			secretsStore.Secrets,
 			solver.NewDockerSocketProvider(),
 		},
 		CacheExports: c.cfg.CacheExports,
@@ -171,12 +171,12 @@ func (c *Client) buildfn(ctx context.Context, st *state.State, env *environment.
 
 	resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
 		s := solver.New(solver.Opts{
-			Control: c.c,
+			Control:      c.c,
-			Gateway: gw,
+			Gateway:      gw,
-			Events:  eventsCh,
+			Events:       eventsCh,
-			Auth:    auth,
+			Auth:         auth,
-			Secrets: secrets,
+			SecretsStore: secretsStore,
-			NoCache: c.cfg.NoCache,
+			NoCache:      c.cfg.NoCache,
 		})
 
 		// Close events channel

environment/pipeline.go

@@ -651,9 +651,27 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S
 		return st, err
 	}
 
-	secret, err := op.Lookup("secret").String()
+	// Inject secret as plain text or retrieve string
+	// FIXME If we could create secret directly in `cue`, we could clean up
+	//  that condition
+	// But currently it's not possible because ECR secret's is a string
+	// so we need to handle both options (string & secret)
+	secretValue, err := op.Lookup("secret").String()
 	if err != nil {
-		return st, err
+		// Retrieve secret
+		if secret := op.Lookup("secret"); secret.Exists() {
+			id, err := getSecretID(secret)
+			if err != nil {
+				return st, err
+			}
+			secretBytes, err := p.s.GetOptions().SecretsStore.GetSecret(ctx, id)
+			if err != nil {
+				return st, err
+			}
+			secretValue = string(secretBytes)
+		} else {
+			return st, err
+		}
 	}
 
 	target, err := op.Lookup("target").String()
@@ -661,7 +679,7 @@ func (p *Pipeline) DockerLogin(ctx context.Context, op *compiler.Value, st llb.S
 		return st, err
 	}
 
-	p.s.AddCredentials(target, username, secret)
+	p.s.AddCredentials(target, username, secretValue)
 	log.
 		Ctx(ctx).
 		Debug().

solver/secretsprovider.go

@@ -11,8 +11,22 @@ import (
 	"go.dagger.io/dagger/state"
 )
 
-func NewSecretsProvider(st *state.State) session.Attachable {
+type SecretsStore struct {
-	return secretsprovider.NewSecretProvider(&inputStore{st})
+	Secrets session.Attachable
+	store   *inputStore
+}
+
+func (s SecretsStore) GetSecret(ctx context.Context, id string) ([]byte, error) {
+	return s.store.GetSecret(ctx, id)
+}
+
+func NewSecretsStoreProvider(st *state.State) SecretsStore {
+	store := &inputStore{st}
+
+	return SecretsStore{
+		Secrets: secretsprovider.NewSecretProvider(store),
+		store:   store,
+	}
 }
 
 type inputStore struct {

solver/solver.go

@@ -26,12 +26,12 @@ type Solver struct {
 }
 
 type Opts struct {
-	Control *bk.Client
+	Control      *bk.Client
-	Gateway bkgw.Client
+	Gateway      bkgw.Client
-	Events  chan *bk.SolveStatus
+	Events       chan *bk.SolveStatus
-	Auth    *RegistryAuthProvider
+	Auth         *RegistryAuthProvider
-	Secrets session.Attachable
+	SecretsStore SecretsStore
-	NoCache bool
+	NoCache      bool
 }
 
 func New(opts Opts) Solver {
@@ -61,6 +61,10 @@ func invalidateCache(def *llb.Definition) error {
 	return nil
 }
 
+func (s Solver) GetOptions() Opts {
+	return s.opts
+}
+
 func (s Solver) NoCache() bool {
 	return s.opts.NoCache
 }
@@ -189,7 +193,7 @@ func (s Solver) Export(ctx context.Context, st llb.State, img *dockerfile2llb.Im
 		Exports: []bk.ExportEntry{output},
 		Session: []session.Attachable{
 			s.opts.Auth,
-			s.opts.Secrets,
+			s.opts.SecretsStore.Secrets,
 			NewDockerSocketProvider(),
 		},
 	}

stdlib/.dagger/env/docker-pull/values.yaml

@@ -3,7 +3,7 @@ plan:
 name: docker-pull
 inputs:
     ref:
-        text: docker.io/daggerio/ci-test:xtyzsocvpici@sha256:35fc94d52b4fa53c2caa38ff11e13182e6f88c651eb0846728d1007d931f0d3c
+        text: docker.io/daggerio/ci-test:pncdyzkdemof@sha256:b92cbbfef6b952befc38812cd88cf5c4c1012f6df2891595c226f56cc053334e
 sops:
     kms: []
     gcp_kms: []
@@ -19,8 +19,8 @@ sops:
             SG1raUVNTzZIWDltV1pOS3hySHlJeWcKg3blmstOGcxtPww513+mAEA0MWOXwNAT
             5ngRvG6MraW3g9dhIuUYOwjuJyz1Z07/DBEocSxnjSyw45ZCkM1/9Q==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2021-07-08T09:53:37Z"
+    lastmodified: "2021-08-31T10:10:02Z"
-    mac: ENC[AES256_GCM,data:kcONOT/cxu39rCrWtMEwHnSZU0o752WyrLMckPp4AGhkQ0CVb7vnnNQ0lLSzsUQfvf0Ze09kFZYuhlGqZ6EPCJvOw0girrdBi09hU2a7Nm8CZd/ku9gP08YsGV3yx0PgIYFuVQRJ60hwQEIZI5neEGV9x2FPUedy9lYbKvvboSE=,iv:ofZ605QYbEbtWNgGxNkp1QbK/VHtwchpFs4GxBU9rIg=,tag:Mo+0nfe2GaJcXpIOCl/cew==,type:str]
+    mac: ENC[AES256_GCM,data:30qNlAVLJunPEboTzeIxcsZ06LcLiDiXXJLVqHE328hcezcOYGsvhlYTiGEzxtAsv78Mwxw54oSbiFZmCKoew9bTZFUyb6FcFVk4GG8z2I8pn7FkZlcnEknWinVf9Tc/h5R/g4/BBGzsBf2dr4fx4ADewwO2z1Df/8wdup0PD4E=,iv:KJcMdpLCfSU1LvvPMXitSPzm0JPwrDWdLncdvVFngNk=,tag:X2/D+RhEnyizZHXJWYnmmg==,type:str]
     pgp: []
     encrypted_suffix: secret
     version: 3.7.1

stdlib/.dagger/env/docker-push-invalid-creds/values.yaml

@@ -3,7 +3,7 @@ plan:
 name: docker-push-invalid-creds
 inputs:
     TestRegistry.secret:
-        text: ENC[AES256_GCM,data:PckymCtA/Q==,iv:to7XhUUcZrWDga7uT4C067BRzHEzmTPDUNAEb2TpS/I=,tag:jUTk8uGd185hmIvi/IHpww==,type:str]
+        secret: ENC[AES256_GCM,data:+gCg3g==,iv:TVQBLFvC1T+xNSJdmhEz+0cciIpCbo6D+twwghUU0ik=,tag:R7SoByjnyj6Aupw1/6c+8w==,type:str]
     TestRegistry.username:
         text: invalid
 sops:
@@ -21,8 +21,8 @@ sops:
             VC8wSTZvUE5UaDg2WE1CaGMzR3M1TEkK9v83AVI4lvFgjKCg8UmQrcxarlESWTfV
             2cDdWgoH7ZqgXo5jFv2tn8qQWHKl8eTTeYUWn8GoNVPKrCroax2fiQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2021-07-08T09:53:54Z"
+    lastmodified: "2021-08-31T10:07:27Z"
-    mac: ENC[AES256_GCM,data:hnVsqFM81iSA/VFPbwqtqw3jOB7H2+67VuXmKfU5fEf15D5WGLZi17HCfRJQ+Db8d0S8ICwFlzqemq+99GB0wf0QVGeOBfrLZ+/AW6Yhd7klhZZxtngXos0lcZreBWduiLkctS2lbx0aiRDBUrsKFcJmu0O9JbMkwC7Hj+nncBk=,iv:2PDO6MTSszlVwmEkAI5lI9cBoJW8JdL3Q+i+sQgtFAk=,tag:nW2OLjAhSot7VyolrXbV5Q==,type:str]
+    mac: ENC[AES256_GCM,data:sdycrW51n0tHL76DroLAUR33Fis5Hixn6dQ7LofNoIcdAj334MTWIf0jxnbzrv4Dkm/MsU90asiGwQyHI56t8mBUqrLJmd8PBE/t6S4RghCAIlM3mcHB4iHsC8Sib2URn3wKztcIuobfU8e9IvZoW4X8R/QWc1jWNmIt8VGdwfw=,iv:g7ri14SRxhsd1SSibYzDig6mZRG7LJ+R6CPDNmNOAfI=,tag:wM1DVa3LL9zFcHKAJJjugg==,type:str]
     pgp: []
     encrypted_suffix: secret
     version: 3.7.1

stdlib/dagger/op/op.cue

@@ -68,7 +68,7 @@ package op
 	target:   string
 	username: string
 	// FIXME: should be a #Secret (circular import)
-	secret: string | bytes
+	secret: _ @dagger(secret)
 }
 
 #FetchContainer: {

stdlib/docker/docker.cue

@@ -8,7 +8,7 @@ import (
 
 // Build a Docker image from source, using included Dockerfile
 #Build: {
-	source: dagger.#Artifact @dagger(input)
+	source: dagger.#Input & {dagger.#Artifact}
 
 	#up: [
 		op.#DockerBuild & {
@@ -21,7 +21,7 @@ import (
 // Pull a docker container
 #Pull: {
 	// Remote ref (example: "index.docker.io/alpine:latest")
-	from: string @dagger(input)
+	from: dagger.#Input & {string}
 
 	#up: [
 		op.#FetchContainer & {ref: from},
@@ -31,18 +31,18 @@ import (
 // Push a docker image to a remote registry
 #Push: {
 	// Remote target (example: "index.docker.io/alpine:latest")
-	target: string @dagger(input)
+	target: dagger.#Input & {string}
 
 	// Image source
-	source: dagger.#Artifact @dagger(input)
+	source: dagger.#Input & {dagger.#Artifact}
 
 	// Registry auth
 	auth?: {
 		// Username
-		username: string @dagger(input)
+		username: dagger.#Input & {string}
 
 		// Password or secret
-		secret: string @dagger(input)
+		secret: dagger.#Input & {dagger.#Secret | string}
 	}
 
 	push: #up: [
@@ -72,7 +72,7 @@ import (
 				source: "/image_ref"
 			},
 		]
-	} @dagger(output)
+	} & dagger.#Output
 
 	// Image digest
 	digest: {
@@ -85,43 +85,43 @@ import (
 				source: "/image_digest"
 			},
 		]
-	} @dagger(output)
+	} & dagger.#Output
 }
 
 #Run: {
 	// Connect to a remote SSH server
 	ssh: {
 		// ssh host
-		host: string @dagger(input)
+		host: dagger.#Input & {string}
 
 		// ssh user
-		user: string @dagger(input)
+		user: dagger.#Input & {string}
 
 		// ssh port
-		port: *22 | int @dagger(input)
+		port: dagger.#Input & {*22 | int}
 
 		// private key
-		key: dagger.#Secret @dagger(input)
+		key: dagger.#Input & {dagger.#Secret}
 
 		// fingerprint
-		fingerprint?: string @dagger(input)
+		fingerprint?: dagger.#Input & {string}
 
 		// ssh key passphrase
-		keyPassphrase?: dagger.#Secret @dagger(input)
+		keyPassphrase?: dagger.#Input & {dagger.#Secret}
 	}
 
 	// Image reference (e.g: nginx:alpine)
-	ref: string @dagger(input)
+	ref: dagger.#Input & {string}
 
 	// Container name
-	name?: string @dagger(input)
+	name?: dagger.#Input & {string}
 
 	// Image registry
 	registry?: {
 		target:   string
 		username: string
 		secret:   dagger.#Secret
-	} @dagger(input)
+	} & dagger.#Input
 
 	#command: #"""
 		# Run detach container
@@ -150,10 +150,10 @@ import (
 // FIXME: incorporate into #Build
 #ImageFromDockerfile: {
 	// Dockerfile passed as a string
-	dockerfile: string @dagger(input)
+	dockerfile: dagger.#Input & {string}
 
 	// Build context
-	context: dagger.#Artifact @dagger(input)
+	context: dagger.#Input & {dagger.#Artifact}
 
 	#up: [
 		op.#DockerBuild & {

stdlib/docker/tests/push-invalid-creds/push.cue

@@ -1,12 +1,13 @@
 package docker
 
 import (
+	"alpha.dagger.io/dagger"
 	"alpha.dagger.io/random"
 )
 
 TestRegistry: {
-	username: string @dagger(input)
+	username: dagger.#Input & {string}
-	secret:   string @dagger(input)
+	secret:   dagger.#Input & {dagger.#Secret}
 }
 
 TestPush: {

stdlib/docker/tests/push-multi-registry/push.cue

@@ -4,9 +4,7 @@ import (
 	"alpha.dagger.io/aws"
 	"alpha.dagger.io/aws/ecr"
 	"alpha.dagger.io/dagger"
-	"alpha.dagger.io/dagger/op"
 	"alpha.dagger.io/random"
-	"alpha.dagger.io/alpine"
 )
 
 // 
@@ -48,36 +46,10 @@ TestRemoteAWS: {
 	}
 }
 
-#TestGetSecret: {
-	secret: dagger.#Artifact
-
-	out: {
-		string
-
-		#up: [
-			op.#Load & {from: alpine.#Image},
-
-			op.#Exec & {
-				always: true
-				args: ["sh", "-c", "cp /input/secret /secret"]
-				mount: "/input/secret": "secret": secret
-			},
-
-			op.#Export & {
-				source: "/secret"
-			},
-		]
-	}
-}
-
 TestRemoteDocker: {
 	dockerConfig: {
-		username: string & dagger.#Input
+		username: dagger.#Input & {string}
-		secret:   dagger.#Secret & dagger.#Input
+		secret:   dagger.#Input & {dagger.#Secret}
-	}
-
-	secret: #TestGetSecret & {
-		secret: dockerConfig.secret
 	}
 
 	target: "daggerio/ci-test:test-docker-\(TestResources.suffix.out)"
@@ -87,7 +59,7 @@ TestRemoteDocker: {
 		source:   TestResources.image
 		auth: {
 			username: dockerConfig.username
-			"secret": secret.out
+			secret:   dockerConfig.secret
 		}
 	}
 }

stdlib/docker/tests/push/push.cue

@@ -1,37 +1,13 @@
 package docker
 
 import (
-	"alpha.dagger.io/dagger/op"
 	"alpha.dagger.io/dagger"
-	"alpha.dagger.io/alpine"
 	"alpha.dagger.io/random"
 )
 
 TestRegistry: {
-	username: string         @dagger(input)
+	username: dagger.#Input & {string}
-	secret:   dagger.#Secret @dagger(input)
+	secret:   dagger.#Input & {dagger.#Secret}
-}
-
-#TestGetSecret: {
-	secret: dagger.#Artifact
-
-	out: {
-		string
-
-		#up: [
-			op.#Load & {from: alpine.#Image},
-
-			op.#Exec & {
-				always: true
-				args: ["sh", "-c", "cp /input/secret /secret"]
-				mount: "/input/secret": "secret": secret
-			},
-
-			op.#Export & {
-				source: "/secret"
-			},
-		]
-	}
 }
 
 TestPush: {
@@ -41,10 +17,6 @@ TestPush: {
 
 	target: "daggerio/ci-test:\(tag.out)"
 
-	secret: #TestGetSecret & {
-		secret: TestRegistry.secret
-	}
-
 	image: #ImageFromDockerfile & {
 		dockerfile: """
 				FROM alpine
@@ -58,7 +30,7 @@ TestPush: {
 		source:   image
 		auth: {
 			username: TestRegistry.username
-			"secret": secret.out
+			secret:   TestRegistry.secret
 		}
 	}
 }

tests/ops.bats

@@ -81,14 +81,8 @@ setup() {
 }
 
 @test "op.#PushContainer" {
-    skip_unless_secrets_available "$TESTDIR"/ops/push-container/inputs.yaml
+    dagger_new_with_env "$TESTDIR"/ops/push-container/
-
+    run "$DAGGER" up -e push-container
-    # ensure the tests fail without credentials
-    run "$DAGGER" compute "$TESTDIR"/ops/push-container/valid
-    assert_failure
-
-    # check that they succeed with the credentials
-    run "$DAGGER" compute --input-yaml "$TESTDIR"/ops/push-container/inputs.yaml "$TESTDIR"/ops/push-container
     assert_success
 }
 

tests/ops/push-container/.dagger/env/push-container/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

tests/ops/push-container/.dagger/env/push-container/values.yaml

@@ -0,0 +1,28 @@
+plan:
+    package: .
+name: push-container
+inputs:
+    registry.secret:
+        secret: ENC[AES256_GCM,data:kZivIXveEoh3+GayI1Tnn6zVf3e38Hlsd2V/GoeJ5bbMOiGS,iv:am3fLZE39MkfFlTXF90olpx/gkp/g1Olkf3PHDsAcKk=,tag:holrnoNR4SqJuw1lYgnUGw==,type:str]
+    registry.username:
+        text: daggertest
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyRzBHQkNGdGpuVWUzeGdl
+            WXhYeGVVeUdFbTZLYWt2enFuUmlHTC83NDBZClh1NSthd3dDdGlaQVVUY09WRmNN
+            cUVPZUY0U3kvZm8zUVZMMjNMMEZrNFEKLS0tIHZYUE8zTTUwUGJWcWtweEo2R09q
+            UVBXdTZSSHhKRGlpOGErd1ZYL0JKY3MKEcay8Bhy1Ap9Jt8gq/6bV/VnzlPSxqZk
+            bd2SuoUs7rMiabhdbApym0MmghqNrwA8xQQo4dCWnxPGDa3avKtTCg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-08-30T15:05:57Z"
+    mac: ENC[AES256_GCM,data:bfTPWrvxKB9a6bc2CPC9vE4BSd/H0JNMx1Cj8JT7KIb82Eymby/B+MEBQTDsVSaILEK6JUgH7Awl/JZAw1MwKG8K8nSAoX74311lRXXnYUNaggXgJu+V4cfdCZuofivQyymsZITMVfABmmjRoZ4zqWiJvYvqdtb42abRMU3vIpA=,iv:WvBCkZXyxYEM4HRdSiCfHdcK/olJOdq2uJpFE2R9wDU=,tag:BemiMETEs1ZmfUUrUUZTjA==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

tests/ops/push-container/inputs.yaml

@@ -1,23 +0,0 @@
-registry:
-    username: ENC[AES256_GCM,data:Yuv+E9dhGZnCxw==,iv:ezThCQJv+bVBf8SdfSa2HFoP+eu6IZMPl5xvMOGDcps=,tag:8+EeJfySzwMczqrzIEDy+w==,type:str]
-    secret: ENC[AES256_GCM,data:OcxwHjWcTdtyKRb7whgG/fzmIG/bpQoSlUVIIhyeEX31lGWh,iv:YAXcRzBoemmef5PBdAOBa5acNPo4BoKH7Ngud/CWYfA=,tag:M1M1hsqKP0TyQbkU5c6oGg==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOaE5NYkdtR0FiOWZTcW54
-            UzRTL3lwQ3owV0QyWmYzVmQ1K0swa2xZK0RnClZSblRIQUxpUWNUdGJBMngwRlFT
-            RXI1aHJMUVVySVF2dzBLN1djZitSWlEKLS0tIGd6RWttckdQTVV1Qi9uWUEvQitR
-            ODEwdXlXSy8veWZkNUpNbWszMVE3M0EKSiQ0AVvySOUHg6RZkcbmpLTHSlnT2zw7
-            Em+pRLYs7GXyilGvSwlRw5O+SrNNQU8Tr8/Yumif2Mks5r3TatDqdA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2021-05-28T20:47:36Z"
-    mac: ENC[AES256_GCM,data:tSOZ6GWrpwPkwCYdtN9/Ym9OXGDzLfXaTATBodaLVjxsVtjaFSxjN15gcjtcxU9KNiOo77fJuEgHgQTQmzHrjSBkvX0zgGoNGU1KCQ3XqRMzfjm1yBU7sWb7lCwjAUqzhERRwe9Vja9GkDSgT+B+CUIRDyqQa1jXg0HlQldhEr0=,iv:ZioCDF8NueNw9miTWxhYWvn1cDw9wUxzMIlp9b2UEgE=,tag:CM4mbhrYW83/ijHNRtIWBw==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.7.1

tests/ops/push-container/main.cue

@@ -1,14 +1,15 @@
 package main
 
 import (
+	"alpha.dagger.io/dagger"
 	"alpha.dagger.io/dagger/op"
 	"alpha.dagger.io/alpine"
 	"alpha.dagger.io/random"
 )
 
 registry: {
-	username: string
+	username: dagger.#Input & {string}
-	secret:   string
+	secret:   dagger.#Input & {dagger.#Secret}
 }
 
 TestPushContainer: {