stdlib: Add GCS support

The interface and behavior is identical to S3.

Signed-off-by: Scott Lewis-Kelly <slewiskelly@slewiskel.ly>

docs/reference/universe/README.md

@@ -15,6 +15,7 @@
 - [docker/compose](./docker/compose.md) - Docker-compose operations
 - [gcp](./gcp/README.md) - Google Cloud Platform
 - [gcp/gcr](./gcp/gcr.md) - Google Container Registry
+- [gcp/gcs](./gcp/gcs.md) - Google Cloud Storage
 - [gcp/gke](./gcp/gke.md) - Google Kubernetes Engine
 - [git](./git.md) - Git operations
 - [go](./go.md) - Go build operations

docs/reference/universe/gcp/gcs.md

@@ -0,0 +1,34 @@
+---
+sidebar_label: gcs
+---
+
+# alpha.dagger.io/gcp/gcs
+
+Google Cloud Storage
+
+```cue
+import "alpha.dagger.io/gcp/gcs"
+```
+
+## gcs.#Object
+
+GCS Bucket object(s) sync
+
+### gcs.#Object Inputs
+
+| Name                  | Type                  | Description                                                       |
+| -------------         |:-------------:        |:-------------:                                                    |
+|*config.region*        | `string`              |GCP region                                                         |
+|*config.project*       | `string`              |GCP project                                                        |
+|*config.serviceKey*    | `dagger.#Secret`      |GCP service key                                                    |
+|*source*               | `dagger.#Artifact`    |Source Artifact to upload to GCS                                   |
+|*target*               | `string`              |Target GCS URL (eg. gs://\<bucket-name\>/\<path\>/\<sub-path\>)    |
+|*delete*               | `*false \| true`      |Delete files that already exist on remote destination              |
+|*contentType*          | `*"" \| string`       |Object content type                                                |
+|*always*               | `*true \| false`      |Always write the object to GCS                                     |
+
+### gcs.#Object Outputs
+
+| Name             | Type              | Description                      |
+| -------------    |:-------------:    |:-------------:                   |
+|*url*             | `string`          |URL of the uploaded GCS object    |

stdlib/.dagger/env/google-gcs/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

stdlib/.dagger/env/google-gcs/values.yaml

@@ -0,0 +1,32 @@
+plan:
+    module: .dagger/env/google-gcs/plan
+    package: ./tests
+name: google-gcs
+inputs:
+    TestConfig.gcpConfig.project:
+        text: dagger-ci
+    TestConfig.gcpConfig.region:
+        text: us-west2-a
+    TestDirectory:
+        dir:
+            path: ./gcp/gcs/tests/testdata
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1anekvutsz37ev3vceyg33p7lffulg7twfxmm5mtzeyc3yn336quqq3hq4q
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2MHRnNWlzN1haMENFRWhT
+            MUlUcXNvYU1DUXJzaU9jcS9TemRlZVF0LzNnCjdIVmpYM1lKQVNPRitxbXEyQzg2
+            Si9qZkJZV3B4a2ZZOVE1ODFaa3NQWVUKLS0tIFRqTk5pVERHckEyYnM0NG9CelJV
+            dm45WXJ4NmxOUHAxeThESTBPdXg4c3cKmUZaIzSFHjwaoS+VqJBg1O9PQGNwhI9d
+            9hvBy/BA0XaT5yIDjevjLP9vCAFDYl20G5Am2q939B8colWtic1Qug==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-06-29T23:52:31Z"
+    mac: ENC[AES256_GCM,data:mza0dLYMFGKgMV04YXtzix6QDtGR1PKPp8VQToOIAvBCqbLK8ti8TUs93t1wXl2Hn18StcUkw5SuFxSE+opn51Q/4TbfN726mnigAzWAd6nKbKLGvj4l1Bb/gDE39ENm7IxqoewMYBh1gufnqgKbGUrHFHK4ZOad7XsbF7UH0Pc=,iv:LBu0PAD6knb0SyVokjtF8daiDhQg3WTpCR1rRVIH5qY=,tag:tMpHIjr9pocxi7UIqsGvqg==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

stdlib/gcp/gcloud.cue

@@ -28,6 +28,7 @@ import (
 				#"""
                 curl -sfL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-\#(version)-linux-x86_64.tar.gz | tar -C /usr/local -zx
                 ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/local/bin
+                ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/local/bin
                 """#,
 			]
 		},

stdlib/gcp/gcs/gcs.cue

@@ -0,0 +1,82 @@
+// Google Cloud Storage
+package gcs
+
+import (
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/dagger/op"
+	"alpha.dagger.io/gcp"
+)
+
+// GCS Bucket object(s) sync
+#Object: {
+
+	// GCP Config
+	config: gcp.#Config
+
+	// Source Artifact to upload to GCS
+	source: dagger.#Artifact @dagger(input)
+
+	// Target GCS URL (eg. gs://<bucket-name>/<path>/<sub-path>)
+	target: string @dagger(input)
+
+	// Delete files that already exist on remote destination
+	delete: *false | true @dagger(input)
+
+	// Object content type
+	contentType: string | *"" @dagger(input)
+
+	// Always write the object to GCS
+	always: *true | false @dagger(input)
+
+	// URL of the uploaded GCS object
+	url: {
+		string
+
+		#up: [
+			op.#Load & {
+				from: gcp.#GCloud & {
+					"config": config
+				}
+			},
+
+			op.#Exec & {
+				if always {
+					always: true
+				}
+				env: {
+					TARGET:           target
+					OPT_CONTENT_TYPE: contentType
+					if delete {
+						OPT_DELETE: "1"
+					}
+				}
+
+				mount: "/source": from: source
+
+				args: [
+					"/bin/bash",
+					"--noprofile",
+					"--norc",
+					"-eo",
+					"pipefail",
+					"-c",
+					#"""
+						opts=("-r")
+						[ -n "$OPT_CONTENT_TYPE" ] && opts+="-h Content-Type:$OPT_CONTENT_TYPE"
+						[ -n "$OPT_DELETE" ] && opts+="-d"
+						gsutil rsync ${opts[@]} /source "$TARGET"
+						echo -n "$TARGET" \
+							| sed -E 's=^gs://([^/]*)/=https://storage.cloud.google.com/\1/=' \
+							> /url
+						"""#,
+
+				]
+			},
+
+			op.#Export & {
+				source: "/url"
+				format: "string"
+			},
+		]
+	} @dagger(output)
+}

stdlib/gcp/gcs/tests/gcs.cue

@@ -0,0 +1,33 @@
+package gcs
+
+import (
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/gcp"
+)
+
+TestConfig: gcpConfig: gcp.#Config
+
+bucket: "dagger-ci"
+
+TestDirectory: dagger.#Artifact
+
+TestGCSObject: {
+	deploy: #Object & {
+		always: true
+		config: TestConfig.gcpConfig
+		source: TestDirectory
+		target: "gs://\(bucket)/"
+	}
+
+	verifyFile: #VerifyGCS & {
+		config: TestConfig.gcpConfig
+		target: deploy.target
+		file:   "dirFile.txt"
+	}
+
+	verifyDir: #VerifyGCS & {
+		config: TestConfig.gcpConfig
+		target: deploy.target
+		file:   "foo.txt"
+	}
+}

stdlib/gcp/gcs/tests/testdata/bar/foo.txt

@@ -0,0 +1 @@
+Test recursivity

stdlib/gcp/gcs/tests/testdata/dirFile.txt

@@ -0,0 +1 @@
+Test directory

stdlib/gcp/gcs/tests/verify.cue

@@ -0,0 +1,83 @@
+package gcs
+
+import (
+	"alpha.dagger.io/alpine"
+	"alpha.dagger.io/dagger/op"
+	"alpha.dagger.io/gcp"
+)
+
+#List: {
+	// GCP Config
+	config: gcp.#Config
+
+	// Target GCP URL (e.g. gs://<bucket-name>/<path>/<sub-path>)
+	target?: string
+
+	contents: {
+		string
+
+		#up: [
+			op.#Load & {
+				from: #GCloud & {
+					"config": config
+				}
+			},
+
+			op.#Exec & {
+				args: [
+					"/bin/bash",
+					"--noprofile",
+					"--norc",
+					"-eo",
+					"pipefail",
+					"-c",
+					#"""
+					gsutil ls -r \#(target) > /contents
+					"""#,
+				]
+			},
+
+			op.#Export & {
+				source: "/contents"
+				format: "string"
+			},
+		]
+	}
+}
+
+#VerifyGCS: {
+	file:   string
+	config: gcp.#Config
+	target: string
+
+	lists: #List & {
+		"config": config
+		"target": target
+	}
+
+	test: #up: [
+		op.#Load & {
+			from: alpine.#Image & {
+				package: bash: "~5.1"
+			}
+		},
+
+		op.#WriteFile & {
+			dest:    "/test"
+			content: lists.contents
+		},
+
+		op.#Exec & {
+			always: true
+			args: [
+				"/bin/bash",
+				"--noprofile",
+				"--norc",
+				"-eo",
+				"pipefail",
+				"-c",
+				"grep -q \(file) /test",
+			]
+		},
+	]
+}

stdlib/universe.bats

@@ -121,6 +121,10 @@ setup() {
     dagger -e google-gcr up
 }
 
+# @test "google cloud: gcs" {
+#     dagger -e google-gcs up
+# }
+
 @test "google cloud: gke" {
     dagger -e google-gke up
 }