Merge pull request #1272 from talentedmrjones/europa-git-pull

ported op.#FetchGit to engine.#GitPull

docs/reference/europa/dagger/engine.md

@@ -92,7 +92,7 @@ _No output._
 
 ## engine.#GitPull
 
-Pull a directory from a git remote
+Pull a directory from a git remote Warning: do NOT embed credentials in the remote url as this will expose them in logs. By using username and password Dagger will handle this for you in a secure manner.
 
 ### engine.#GitPull Inputs
 

plan/task/gitpull.go

@@ -0,0 +1,95 @@
+package task
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/compiler"
+	"go.dagger.io/dagger/plancontext"
+	"go.dagger.io/dagger/solver"
+)
+
+func init() {
+	Register("GitPull", func() Task { return &gitPullTask{} })
+}
+
+type gitPullTask struct {
+}
+
+func (c gitPullTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+	var gitPull struct {
+		Remote     string
+		Ref        string
+		KeepGitDir bool
+		Auth       struct {
+			Username string
+		}
+	}
+
+	if err := v.Decode(&gitPull); err != nil {
+		return nil, err
+	}
+
+	gitOpts := []llb.GitOption{}
+
+	lg := log.Ctx(ctx)
+
+	if gitPull.KeepGitDir {
+		lg.Debug().Str("keepGitDir", "true").Msg("adding git option")
+		gitOpts = append(gitOpts, llb.KeepGitDir())
+	}
+
+	if gitPull.Auth.Username != "" {
+		pwd := v.Lookup("auth.password")
+
+		pwdSecret, err := pctx.Secrets.FromValue(pwd)
+		if err != nil {
+			return nil, err
+		}
+
+		remote, err := url.Parse(gitPull.Remote)
+		if err != nil {
+			return nil, err
+		}
+
+		lg.Debug().Str("username", gitPull.Auth.Username).Str("password", "***").Msg("using username:password auth")
+		remote.User = url.UserPassword(gitPull.Auth.Username, strings.TrimSpace(pwdSecret.PlainText()))
+		gitPull.Remote = remote.String()
+	} else if authToken := v.Lookup("auth.authToken"); plancontext.IsSecretValue(authToken) {
+		authTokenSecret, err := pctx.Secrets.FromValue(authToken)
+		if err != nil {
+			return nil, err
+		}
+		lg.Debug().Str("authToken", "***").Msg("adding git option")
+		gitOpts = append(gitOpts, llb.AuthTokenSecret(authTokenSecret.ID()))
+	} else if authHeader := v.Lookup("auth.authHeader"); plancontext.IsSecretValue(authHeader) {
+		authHeaderSecret, err := pctx.Secrets.FromValue(authHeader)
+		if err != nil {
+			return nil, err
+		}
+		lg.Debug().Str("authHeader", "***").Msg("adding git option")
+		gitOpts = append(gitOpts, llb.AuthHeaderSecret(authHeaderSecret.ID()))
+	}
+
+	remoteRedacted := gitPull.Remote
+	if u, err := url.Parse(gitPull.Remote); err == nil {
+		remoteRedacted = u.Redacted()
+	}
+
+	gitOpts = append(gitOpts, withCustomName(v, "GitPull %s@%s", remoteRedacted, gitPull.Ref))
+
+	st := llb.Git(gitPull.Remote, gitPull.Ref, gitOpts...)
+
+	result, err := s.Solve(ctx, st, pctx.Platform.Get())
+	if err != nil {
+		return nil, err
+	}
+
+	fs := pctx.FS.New(result)
+	return compiler.NewValue().FillFields(map[string]interface{}{
+		"output": fs.MarshalCUE(),
+	})
+}

stdlib/europa/dagger/engine/git.cue

@@ -11,11 +11,20 @@ package engine
 }
 
 // Pull a directory from a git remote
+// Warning: do NOT embed credentials in the remote url as this will expose them in logs. 
+// By using username and password Dagger will handle this for you in a secure manner.
 #GitPull: {
-	@dagger(notimplemented)
 	$dagger: task: _name: "GitPull"
-
+	remote:     string
-	remote: string
+	ref:        string
-	ref:    string
+	keepGitDir: true | *false
+	auth?:      {
+		username: string
+		password: #Secret // can be password or personal access token
+	} | {
+		authToken: #Secret
+	} | {
+		authHeader: #Secret
+	}
 	output: #FS
 }

tests/secrets_sops.yaml

@@ -0,0 +1,21 @@
+TestPAT: ENC[AES256_GCM,data:KYPnJTTCaEbEiBwODMDmOmZGx/Vu/4mOZPfRSjhBc239fPfHzDH75w==,iv:j9UFKfdRMfYg/3xw4dCoWbs0Zoy3czqRznlQrRvf4Sc=,tag:Pvd4UMFDOj8rLOwZJjsYpA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUEhWbjV3M29oUUJyWk81
+            Wk1WQ1E0cmtuVlhNSGxkWUM3WmJXdUYvbzAwCjlFWW9IVmtmTjY1aU1LR2lxWFlT
+            am9RemNqSDRWK2FDYk1xeGNiTFlWMFUKLS0tIFVrSzBCMERQbnhYb09ReVpFK00v
+            TG5YUDlFVzlRRFBCdEhsNVlVK1dMRTgKx1TPZWWQiaU8iMni03/ekG+m4rFCcaa4
+            JI+ED2d+8411BgZtlss/ukQtwskidvYTvetyWw2jes6o1lhfDv5q2A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-12-22T22:23:53Z"
+    mac: ENC[AES256_GCM,data:twjo6LyD0QFJoD1h5qh8SFy4XC5+2JHl6mUiUxOjLnSBwft1Ntnto3+2tzZBU6+o+v/Uo++vRoSrIkLARFt74x2xA7jZjc81e5yCkcS79bWSpZ8bA7e8/5hgkyYP5SlMAYsWKXPodmhgN7bwCa6vjZb4ZlFkJuBZrHtLeziVY1E=,iv:kgDj5lnLTgLPK38CcUVYdtE11d/gpFdHPP9hgoZQO9U=,tag:tP2E3t/jKoh2Fx6Z2X/+Tw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

tests/tasks.bats

@@ -93,3 +93,18 @@ setup() {
     "$DAGGER" --europa up ./scratch.cue -l debug
 }
 
+@test "task: #GitPull" {
+    cd "$TESTDIR"
+    "$DAGGER" --europa up ./tasks/gitPull/exists.cue
+    "$DAGGER" --europa up ./tasks/gitPull/git_dir.cue
+    "$DAGGER" --europa up ./tasks/gitPull/private_repo.cue
+
+    run "$DAGGER" --europa up ./tasks/gitPull/invalid.cue
+    assert_failure
+    run "$DAGGER" --europa up ./tasks/gitPull/bad_remote.cue
+    assert_failure
+    run "$DAGGER" --europa up ./tasks/gitPull/bad_ref.cue
+    assert_failure
+
+
+}

tests/tasks/gitPull/bad_ref.cue

@@ -0,0 +1,10 @@
+package main
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	actions: badref: engine.#GitPull & {
+		remote: "https://github.com/blocklayerhq/acme-clothing.git"
+		ref:    "lalalalal"
+	}
+}

tests/tasks/gitPull/bad_remote.cue

@@ -0,0 +1,10 @@
+package main
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	actions: badremote: engine.#GitPull & {
+		remote: "https://github.com/blocklayerhq/lalalala.git"
+		ref:    "master"
+	}
+}

tests/tasks/gitPull/exists.cue

@@ -0,0 +1,10 @@
+package main
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	actions: gitPull: engine.#GitPull & {
+		remote: "https://github.com/blocklayerhq/acme-clothing.git"
+		ref:    "master"
+	}
+}

tests/tasks/gitPull/git_dir.cue

@@ -0,0 +1,36 @@
+package testing
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	actions: {
+		repo1: engine.#GitPull & {
+			remote: "https://github.com/blocklayerhq/acme-clothing.git"
+			ref:    "master"
+		}
+
+		repo2: engine.#GitPull & {
+			remote:     "https://github.com/blocklayerhq/acme-clothing.git"
+			ref:        "master"
+			keepGitDir: true
+		}
+
+		image: engine.#Pull & {
+			source: "alpine:3.15.0"
+		}
+
+		verify: engine.#Exec & {
+			input: image.output
+			args: ["sh", "-c", """
+				set -eu
+				[ ! -d /repo1/.git ]
+				[ -d /repo2/.git ]
+				"""]
+			mounts: {
+				a: {dest: "/repo1", contents: repo1.output}
+				b: {dest: "/repo2", contents: repo2.output}
+			}
+		}
+
+	}
+}

tests/tasks/gitPull/invalid.cue

@@ -0,0 +1,7 @@
+package main
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	actions: invalid: engine.#GitPull & {}
+}

tests/tasks/gitPull/private_repo.cue

@@ -0,0 +1,35 @@
+package main
+
+import "alpha.dagger.io/europa/dagger/engine"
+
+engine.#Plan & {
+	inputs: secrets: token: command: {
+		name: "sops"
+		args: ["exec-env", "./secrets_sops.yaml", "echo $TestPAT"]
+	}
+
+	actions: {
+		alpine: engine.#Pull & {
+			source: "alpine:3.15.0"
+		}
+
+		testRepo: engine.#GitPull & {
+			remote: "https://github.com/dagger/dagger.git"
+			ref:    "main"
+			auth: {
+				username: "dagger-test"
+				password: inputs.secrets.token.contents
+			}
+		}
+
+		testContent: engine.#Exec & {
+			input:  alpine.output
+			always: true
+			args: ["ls", "-l", "/repo/README.md"]
+			mounts: inputRepo: {
+				dest:     "/repo"
+				contents: testRepo.output
+			}
+		}
+	}
+}