Merge pull request #920 from TomChv/feat/git-private-repo

Fetch private git repository

docs/reference/universe/git.md

@@ -37,6 +37,8 @@ A git repository
 |*remote*          | `string`             |Git remote. Example: `"https://github.com/dagger/dagger"`    |
 |*ref*             | `string`             |Git ref: can be a commit, tag or branch. Example: "main"     |
 |*subdir*          | `*null \| string`    |(optional) Subdirectory                                      |
+|*authToken*       | `dagger.#Secret`     |(optional) Add Personal Access Token                         |
+|*authHeader*      | `dagger.#Secret`     |(optional) Add OAuth Token                                   |
 
 ### git.#Repository Outputs
 

environment/pipeline.go

@@ -494,17 +494,11 @@ func (p *Pipeline) mount(ctx context.Context, dest string, mnt *compiler.Value)
 	}
 	// eg. mount: "/foo": secret: mysecret
 	if secret := mnt.Lookup("secret"); secret.Exists() {
-		if !secret.HasAttr("secret") {
+		id, err := getSecretID(secret)
-			return nil, fmt.Errorf("invalid secret %q: not a secret", secret.Path().String())
-		}
-		idValue := secret.Lookup("id")
-		if !idValue.Exists() {
-			return nil, fmt.Errorf("invalid secret %q: no id field", secret.Path().String())
-		}
-		id, err := idValue.String()
 		if err != nil {
-			return nil, fmt.Errorf("invalid secret id: %w", err)
+			return nil, err
 		}
+
 		return llb.AddSecret(dest,
 			llb.SecretID(id),
 			llb.SecretFileOpt(0, 0, 0400), // uid, gid, mask)
@@ -779,6 +773,21 @@ func (p *Pipeline) PushContainer(ctx context.Context, op *compiler.Value, st llb
 	return st, err
 }
 
+func getSecretID(secretField *compiler.Value) (string, error) {
+	if !secretField.HasAttr("secret") {
+		return "", fmt.Errorf("invalid secret %q: not a secret", secretField.Path().String())
+	}
+	idValue := secretField.Lookup("id")
+	if !idValue.Exists() {
+		return "", fmt.Errorf("invalid secret %q: no id field", secretField.Path().String())
+	}
+	id, err := idValue.String()
+	if err != nil {
+		return "", fmt.Errorf("invalid secret id: %w", err)
+	}
+	return id, nil
+}
+
 func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.State) (llb.State, error) {
 	remote, err := op.Lookup("remote").String()
 	if err != nil {
@@ -796,8 +805,6 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.Stat
 
 	gitOpts := []llb.GitOption{}
 	var opts struct {
-		AuthTokenSecret  string
-		AuthHeaderSecret string
 		KeepGitDir bool
 	}
 
@@ -808,11 +815,20 @@ func (p *Pipeline) FetchGit(ctx context.Context, op *compiler.Value, st llb.Stat
 	if opts.KeepGitDir {
 		gitOpts = append(gitOpts, llb.KeepGitDir())
 	}
-	if opts.AuthTokenSecret != "" {
+	// Secret
-		gitOpts = append(gitOpts, llb.AuthTokenSecret(opts.AuthTokenSecret))
+	if authToken := op.Lookup("authToken"); authToken.Exists() {
+		id, err := getSecretID(authToken)
+		if err != nil {
+			return st, err
 		}
-	if opts.AuthHeaderSecret != "" {
+		gitOpts = append(gitOpts, llb.AuthTokenSecret(id))
-		gitOpts = append(gitOpts, llb.AuthTokenSecret(opts.AuthHeaderSecret))
+	}
+	if authHeader := op.Lookup("authHeader"); authHeader.Exists() {
+		id, err := getSecretID(authHeader)
+		if err != nil {
+			return st, err
+		}
+		gitOpts = append(gitOpts, llb.AuthHeaderSecret(id))
 	}
 
 	gitOpts = append(gitOpts, llb.WithCustomName(p.vertexNamef("FetchGit %s@%s", remoteRedacted, ref)))

stdlib/.dagger/env/git/values.yaml

@@ -1,6 +1,9 @@
 plan:
     package: ./git/tests
 name: git
+inputs:
+    TestPAT:
+        secret: ENC[AES256_GCM,data:7s1tSIpIDNBhAFupdjb7KtPbjKrCd5tXupr3RQF2N3Xu5XGuTZMgoQ==,iv:I+SVYLnjgMffvNg6BMB6m1lj+VVH5sDK0aIEAWPcyLY=,tag:TcfJ6LVps8dXVZGZy3T2ew==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -16,8 +19,8 @@ sops:
             TmhJNisyamw3d244aGVJSEVFVUVLZGsKvd+nowA0CLXQbdvyI4J0lBjs9vdISWlo
             gGvR49uul3Z8raVWXFUzsyQ8xTvYNg0ovynFG2KdagSKr1DlhKMBEQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2021-07-08T09:54:31Z"
+    lastmodified: "2021-08-26T13:44:11Z"
-    mac: ENC[AES256_GCM,data:pFrhyJQLJ1zQJmXQWQtmkeraiTHCKvOEr+TVgYQ6EZsei+dL+VUVWDgeHLkonxwh9eBPyAtB1cfxPc+1xVnMCqmFPVZMZ0P+CNgaOTcHk38UzOHyCcjw18AjROuEYffat8XbmjwKaSX+XRvMiC53BTrZkXt6os7hfikrySEot3A=,iv:W9S+qlvAB3gXFhUTpE17Fm/lQK6DTo7mmdzL3LjCVWQ=,tag:/fETJit+AXZ/OjIjz0TPhA==,type:str]
+    mac: ENC[AES256_GCM,data:ttmpbzhrVFEGh/oJF4TtMvf99rutPBbzp9cIaqakIl+5nxqOkuAakgvf7IIMBG235zdyMvIXZZh6NLYG51PZA1hKNMg5Pqqba9GOSvFCHasWzNJ3pi5SLBGD02ivDfkSMbEHeOCUhnG1X4LxkYL9j+fb4tQt1Btv1hiIAcIa+eY=,iv:WxuW+0yJYtNqAB0y1nji9c3lzn4Pftir8uZojcdphng=,tag:yvcIJxkuqOmCfXoyEnGWow==,type:str]
     pgp: []
     encrypted_suffix: secret
     version: 3.7.1

stdlib/dagger/op/op.cue

@@ -87,8 +87,8 @@ package op
 	ref:         string
 	keepGitDir?: bool
 	// FIXME: the two options are currently ignored until we support buildkit secrets
-	authTokenSecret?:  string | bytes
+	authToken?:  _ @dagger(secret)
-	authHeaderSecret?: string | bytes
+	authHeader?: _ @dagger(secret)
 }
 
 #FetchHTTP: {

stdlib/git/git.cue

@@ -11,18 +11,24 @@ import (
 #Repository: {
 	// Git remote.
 	// Example: `"https://github.com/dagger/dagger"`
-	remote: string & dagger.#Input
+	remote: dagger.#Input & {string}
 
 	// Git ref: can be a commit, tag or branch.
 	// Example: "main"
-	ref: string & dagger.#Input
+	ref: dagger.#Input & {string}
 
 	// (optional) Subdirectory
-	subdir: *null | string & dagger.#Input
+	subdir: dagger.#Input & {*null | string}
 
 	// (optional) Keep .git directory
 	keepGitDir: *false | bool
 
+	// (optional) Add Personal Access Token
+	authToken: dagger.#Input & {*null | dagger.#Secret}
+
+	// (optional) Add OAuth Token
+	authHeader: dagger.#Input & {*null | dagger.#Secret}
+
 	#up: [
 		op.#FetchGit & {
 			"remote": remote
@@ -30,6 +36,12 @@ import (
 			if (keepGitDir) {
 				keepGitDir: true
 			}
+			if (authToken != null) {
+				"authToken": authToken
+			}
+			if (authHeader != null) {
+				"authHeader": authHeader
+			}
 		},
 		if subdir != null {
 			op.#Subdir & {

stdlib/git/tests/git.cue

@@ -3,8 +3,9 @@ package git
 import (
 	"strings"
 
-	"alpha.dagger.io/git"
 	"alpha.dagger.io/alpine"
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/git"
 	"alpha.dagger.io/os"
 )
 
@@ -74,3 +75,25 @@ TestCurrentTags: os.#Container & {
 		[ $TAGS = "0=master" ]
 		"""
 }
+
+// Test fetching a private repo
+TestPAT: dagger.#Input & {dagger.#Secret}
+
+privateRepo: git.#Repository & {
+	remote:     "https://github.com/dagger/dagger.git"
+	ref:        "main"
+	keepGitDir: true
+	authToken:  TestPAT
+}
+
+TestPrivateRepository: os.#Container & {
+	image: alpine.#Image & {
+		package: bash: "=5.1.0-r0"
+		package: git:  true
+	}
+	mount: "/repo1": from: privateRepo
+	dir: "/repo1"
+	command: """
+		[ -d .git ]
+		"""
+}

stdlib/universe.bats

@@ -8,7 +8,6 @@ setup() {
     dagger -e sanity-check up
 }
 
-
 @test "os" {
     dagger -e os up
 }

tests/helpers.bash

@@ -26,6 +26,13 @@ dagger_new_with_plan() {
     "$DAGGER" new "$name"
 }
 
+dagger_new_with_env() {
+    local sourcePlan="$1"
+    
+    "$DAGGER" init -w "$DAGGER_WORKSPACE"
+    rsync -av "$sourcePlan"/ "$DAGGER_WORKSPACE"
+}
+
 # dagger helper to execute the right binary
 dagger() {
     "${DAGGER}" "$@"

tests/ops.bats

@@ -108,6 +108,10 @@ setup() {
     run "$DAGGER" compute "$TESTDIR"/ops/fetch-git/gitdir
     assert_success
 
+    dagger_new_with_env "$TESTDIR"/ops/fetch-git/private-repo
+    run "$DAGGER" up -e op-fetch-git
+    assert_success
+
     # FIXME: distinguish missing inputs from incorrect config
     # run "$DAGGER" compute "$TESTDIR"/ops/fetch-git/invalid
     # assert_failure

tests/ops/fetch-git/private-repo/.dagger/env/op-fetch-git/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

tests/ops/fetch-git/private-repo/.dagger/env/op-fetch-git/values.yaml

@@ -0,0 +1,26 @@
+plan:
+    package: .
+name: op-fetch-git
+inputs:
+    TestPAT:
+        secret: ENC[AES256_GCM,data:TVMwgMe+Q/BzQ/rxKIr9lRPuu2FpVQXppufy33lWQ8An+c9cTSVLCQ==,iv:Xpe54Llfcu1aTWkzNxUtcRrrqIlI/i4pdshOCVKeREY=,tag:Ugwzcpmddzhq62gPKpAkkQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoamxhWnpyQzJRcE9TRldj
+            Rmd6eHV5TnpReENzNHI3MFZRZWpqaGJoRzFjClMyQW1raVNiSU84blhhSzRja01F
+            cjVpVmxJS1o0NEdlTXYvQ290Rjdma0kKLS0tIDkrTjRHTEtaaDY2QzhBaVJYRjhn
+            OXA2TkFjYlVNMFFEblA2MUZRWVB3T3MKhunM53KD+jGvdAInPbr+N6YrpirLIp4V
+            seb6c4bryHPL+zZWSX1H04sCWgGOypLfYq/fUyPOunbs/b+AWn8chw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-08-26T09:34:51Z"
+    mac: ENC[AES256_GCM,data:JUo5COutVZMqJN2UNhqL842hhGsMdCLOFFo5g4HDFMQLG3M1iQ2M0i8AjnAlPGMukNoF3qr/rTf6FCNdpeXc/oXiOM+q0w9xp+jGup5uplhXUWw+au4YHXbSezxCVtmKCRIzkDBiN5x2zKLg7qBzsqKdzyeBiBI/QAPCoXuTk4g=,iv:m4GKvZ8g4dniAbTnUO+wjZlYx6uCJcCwkanoAzjciuY=,tag:vUUHFcGg5OIjDHeI0B+RGw==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

tests/ops/fetch-git/private-repo/cue.mod/module.cue

@@ -0,0 +1 @@
+module: ""

tests/ops/fetch-git/private-repo/cue.mod/pkg/.gitignore

@@ -0,0 +1,2 @@
+# dagger universe
+alpha.dagger.io

tests/ops/fetch-git/private-repo/fetch-git.cue

@@ -0,0 +1,22 @@
+package op
+
+import (
+	"alpha.dagger.io/os"
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/dagger/op"
+)
+
+// Github PAT
+TestPAT: dagger.#Input & {dagger.#Secret}
+
+TestRepo: #up: [op.#FetchGit & {
+	remote:    "https://github.com/dagger/dagger.git"
+	ref:       "main"
+	authToken: TestPAT
+}]
+
+TestContent: os.#Container & {
+	always:  true
+	command: "ls -l /input/repo | grep 'universe -> stdlib'"
+	mount: "/input/repo": from: TestRepo
+}