Merge pull request #756 from slewiskelly/gcs

stdlib: Add GCS support
2021-06-30 17:12:54 +02:00 · 2021-06-30 17:12:54 +02:00 · 9c0210d485
commit 9c0210d485
parent fe3779f28f 59281b38fc
11 changed files with 294 additions and 0 deletions
--- a/docs/reference/universe/README.md
+++ b/docs/reference/universe/README.md
@ -15,6 +15,7 @@
 - [docker/compose](./docker/compose.md) - Docker-compose operations
 - [gcp](./gcp/README.md) - Google Cloud Platform
 - [gcp/gcr](./gcp/gcr.md) - Google Container Registry
+- [gcp/gcs](./gcp/gcs.md) - Google Cloud Storage
 - [gcp/gke](./gcp/gke.md) - Google Kubernetes Engine
 - [git](./git.md) - Git operations
 - [go](./go.md) - Go build operations
--- a/docs/reference/universe/gcp/gcs.md
+++ b/docs/reference/universe/gcp/gcs.md
@ -0,0 +1,34 @@
+---
+sidebar_label: gcs
+---
+
+# alpha.dagger.io/gcp/gcs
+
+Google Cloud Storage
+
+```cue
+import "alpha.dagger.io/gcp/gcs"
+```
+
+## gcs.#Object
+
+GCS Bucket object(s) sync
+
+### gcs.#Object Inputs
+
+| Name                  | Type                  | Description                                                       |
+| -------------         |:-------------:        |:-------------:                                                    |
+|*config.region*        | `string`              |GCP region                                                         |
+|*config.project*       | `string`              |GCP project                                                        |
+|*config.serviceKey*    | `dagger.#Secret`      |GCP service key                                                    |
+|*source*               | `dagger.#Artifact`    |Source Artifact to upload to GCS                                   |
+|*target*               | `string`              |Target GCS URL (eg. gs://\<bucket-name\>/\<path\>/\<sub-path\>)    |
+|*delete*               | `*false \| true`      |Delete files that already exist on remote destination              |
+|*contentType*          | `*"" \| string`       |Object content type                                                |
+|*always*               | `*true \| false`      |Always write the object to GCS                                     |
+
+### gcs.#Object Outputs
+
+| Name             | Type              | Description                      |
+| -------------    |:-------------:    |:-------------:                   |
+|*url*             | `string`          |URL of the uploaded GCS object    |
--- a/stdlib/.dagger/env/google-gcs/.gitignore
+++ b/stdlib/.dagger/env/google-gcs/.gitignore
@ -0,0 +1,2 @@
+# dagger state
+state/**
--- a/stdlib/.dagger/env/google-gcs/values.yaml
+++ b/stdlib/.dagger/env/google-gcs/values.yaml
@ -0,0 +1,36 @@
+plan:
+    module: ./gcp/gcs/
+    package: ./tests
+name: google-gcs
+inputs:
+    TestConfig.bucket:
+        text: dagger-ci
+    TestConfig.gcpConfig.project:
+        text: dagger-ci
+    TestConfig.gcpConfig.region:
+        text: us-west2-a
+    TestConfig.gcpConfig.serviceKey:
+        secret: ENC[AES256_GCM,data:UEKTXvyrBgHKOYE9vSGoHua9wWALjghxWu+ui9K3MAS+1mnVlc1qjTbwv/1/hIIkRNlyhY6WlN0k3x2imusFFInzrNZ5G4FJHGiP/zaazd7shUS8LZsh1cL0I1jnsaDJaz4Zw0yVu+FT1z2/+9l81U9MrtvbLNKFSqZJsrymZl5lUCxiRsUEBiC0/rOoOlQ88kfnxUdBXnG7ABciqPUK7cYaMo5RbB1a9YfacB8S2sosClxK727jUgD20I12ru+y5Y/hg00BhBl9bIg35VTI8PFeZvRqQowqaJO+i1BjQbBYef8s9faYdZGEP0hUrvUpPek4Z0ZBDRbxRfRFAiXBbWbvCJErGlsmcNGqllixgDbcYQNKBTjZEKke4PGHtahmPXOIOO9/fxnUCoci//azJ5fUP0Kdiw06DQYQnngwRSA/nOqTBiuWcfiTLY00iQxoh+8Mt5/CMTXhRz8PpRpvXtKLe4ogaVbEBMOXe8+nMtwt0H6kV+YmHgA+vVHw0WYtBF6gmKnXuGWT/dz94SHGe9oMiK7H7KtfRCyN1SBAx3H9R3qitt9114TlWRBvfrVtDsb/E0MQdnbXZQirJ/2ev78DTc5bdRTFus2vvHZB2RP+wr7sB8A+jcWS1RBv0NJOZoPlHqKIfB93NBhI8wU+lqEbTn2Zm0LAvGVypws3ci+0GJFReIok9yGc/WucFBDMWi8tDOwUdbreX7EpdGLSn9Rwq0Oeuo5Udy5Qo4xFAC/v2yCzqb7MrZzX1S+OHyNObUTGuEyTOL8t91KYBebHtLK0Ud9qQoYuO5IvYCGeKIHr5FDFCYKDJAlW8Y9Iw27LsU45rqeHTjUi1BqOLfXZbns79w8WYiz5bqc3eMXmYOgCKtU5BwLo9LV7WR+FW8cCi9gSJHnRbc8uT9V1NWaGeoKuaPXpBWLH/GTKXEKuy1HXz+yPgQsYYN2yrGNgmgXpKbxgp92h6LRA/PRjxrjkAV8kVdivxG3GUFVfq4ZtGlLdRrcYCkDIZVpwPrNp8Qg5NLn5xhWj1hsqqOOJdeDHHLkS48A9pLAhPubRWH0s1IixLstnJ95DGIZDE0QpEh7STctqvS7SVhclroKI5xzT7w3C78fxTKC//JkdrV0/rQ2CQ1u+emaCUt3X1WwYofudoMjp5iESscDvw6PnqzubUT51FwGtAf2hkT07VQWZ2CcNP4dvYgDSsc26Y0hwWdr+ppCy1xbmpENU5moUPELtF9w2da1zG5N32Fcxr9VOqOGwa4pqkasIQPCRdqsEa/34g95L7Z1BAXpMxCLTbW5nxD3wQBsrsTjO0rmBh4n9BdKjqj5GrXOTceAaJ+L3o07aVIzICJ5HGo7GQXjaQOABzysbGr7bdBIQyp//ruqXNLVSaylIunZJY4/HFdqetZj6jfb4rE+/GkjFLw23Ym8RIDYJQsTAR8OdwZNtVPrILdve1ohlyBW/nS+gy5dZ3qHnE9ZxbgN6FpRYr9G4acK3yCyMeuBdiB6Tqz0o6xz/c2WLWpWcw/HivVce53MfbqMhj0K/jjpl+fsJT/+ZkaPviDrOtKR/WPz/ewNwQ7pZgCubtlzCmPw8pBKjP+uI9tvwydgnV8I/LdEvS0IrP9m22kPZiw/m7mPQPkDA7IzQSIlrmWMCt/iJ0rvVEtkyi1pPyzVVby4OyzvyhexVcFu2X1x5jOqfUL3eEjO+IhMJufVCOVlUKNHPci0r4KQZ07n1wPCjUoCN4MMl9D8yaP0l8SQmSmp9/DFVZ1Cd5m+O0cmaQfZX8dYV52uxkPnkRNn9SnKCz5Nf8tBG8p+kvznEYw6d4UKQLocFdzeBU6FBa4zQSf+oXHghds9YGJoWNAGzD9ucgVNyu4hU2rq7UDnAwT/pm8FTbw/OM2uCyPr80J3CpgKPdA0+jEvBNxlezsZeLyTntoZLIstgllETxC9QjqfCVtnQYBYT8oFCXZPadyBZPPVrXvQORCpRXJCQ9iyoiKTQCCOGqGyIgC+23Q0ebmZPr7e4I5JbBZWeh6ek2RRWOZMI0lkLeE72KO+9h8HPxmgMhPtDes/l6DmWjAPb6IwZSM9SseTYF447HL5rQKkv4x7W4Cu1pACSc/2qqdUwnC6jydk8EY2zPa9kdguwdSiDr+KwASZex2MKMJa+peXhTDLqB7jxiO7snU4r0nk7E0B3IBqWM89PVskyNTJXqTkAXyzHWag1S8URPhk9Q9UYpyVCWRzs4nfXAX2WgCJl9iw8ZXrIJZ3fGs3vXQMYI/kGJ7rGIDWhcQEDJGKJYrBCh6zNsbSzBagd5AQ6DNIUEvTUcPvxZP6sy+uiiMMqErDgbi2IT3G6E5sEL0sO7GtrVWoxDQWXX80vtf146fnNNp4uS2wGta7b4kguAt39olceX7nUrYgzLm1aXcr0ObbRjcPdeSlNGLTJGlZRNtvTlFdT1RgTQBfRdG3d1JCraCGsAWxBC+Bwa/zAcdIwUNnMfgdZ5eOAyiBTmoGAAOkUr7avSVvP5Xyrk9JlNzeArrjZUFMYOcXBfkMquqD1NoPHUZ12u9jCoj109zJquxcJ1zGT5TylJYSTc9xz46TfLrz5KvsMhfetBx0ocY8lXjbs3pzB6ZswDGzF4j0eiWBfu9UZoiQVtt1eIfAPoZO9u/n2Wbpx+ov+mJFf0dZklKDED5CUbmj8/izOdE9KbiPDFr1nZZQbO+YwJYhjxxqxzwC41CvH5XQTCCadeWgb5cHEwwunFRVCYeEFvTlP6nR9pS19UZoQvPNk5NEnZVKxWVD1lK18frvX8MxIpB2xaSbUkvGi3tueinLa7OYVfjirBJ1GYbuADa4H0KOxi4MP59ah6TOmsZstA6plFbtRRtk4Iofzu1mAyUN52SZzwCvH5/IxOh6vQxg7EcMfB8O5+S87PeE3vd1ch+UVDwQ3fvNmjd7/77XdKZiR7bwiDxodxABpG4iYrxcfkGPOTCKUlisgFNNmSmwG7FJMYWkz3RcEHib/biklnEKbm/9G5000cwZXXglZb8QjleSklkSEZa4DYec1SNvD/KwRbk5lYr03TDl7DrZo3WjKWvWGMUs3,iv:tPbOGayR7NiXcuHWjX0pX/nSitOxmsr4qqrc6irlIJI=,tag:apejA4UTYTuwT4CUSeoaRQ==,type:str]
+    TestDirectory:
+        dir:
+            path: ./gcp/gcs/tests/testdata
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCRThSTEE1Rm5HU2Y2NkJZ
+            SEd2blAyeEVnMHBqRGxXMEQ3TGFzWTBwd1EwCnA0OFVmTCsxSmpNV29adGt2ZHFH
+            WE9vN1ZoNENFV2t1TGVuZkdwVndNbVUKLS0tIGpHZEptYWxEZVNjcXF4NkoyWHRv
+            ZXd6Qmd1YUtxMnVTVkYybWgrV3pVK2MKowMeOZU3j3BxERT0DwhQYCGUDBK6gCdo
+            WByubiBATdsb7h7ytCC4HutWppynK4MpU+Ya9NP83AZuXo+Wa2u6aQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-06-30T10:44:08Z"
+    mac: ENC[AES256_GCM,data:bVjVtONv7ILxZ3UkYy/l3MnuJB8g4EG++fBfeL/CQPHyembwBURC3nxfB0IdqQmPsa0/ahycNawIvB/owy+Sw6yyaDWGtd/8Jn0CILLZ4Nyx9lNN95J4Ad0FbUKr24Fu17I339rB1GrEjAaTs1qcle53sp4Wq9Blg26Vkut9jmc=,iv:TdFDptBZrScOfywWsIhoLijSFQL18DF94bKMh5DfW5s=,tag:wVoa1K/Nyx05nu+Op0Viuw==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1
--- a/stdlib/gcp/gcloud.cue
+++ b/stdlib/gcp/gcloud.cue
@ -28,6 +28,7 @@ import (
 				#"""
                curl -sfL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-\#(version)-linux-x86_64.tar.gz | tar -C /usr/local -zx
                ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/local/bin
+                ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/local/bin
                """#,
 			]
 		},
--- a/stdlib/gcp/gcs/gcs.cue
+++ b/stdlib/gcp/gcs/gcs.cue
@ -0,0 +1,82 @@
+// Google Cloud Storage
+package gcs
+
+import (
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/dagger/op"
+	"alpha.dagger.io/gcp"
+)
+
+// GCS Bucket object(s) sync
+#Object: {
+
+	// GCP Config
+	config: gcp.#Config
+
+	// Source Artifact to upload to GCS
+	source: dagger.#Artifact @dagger(input)
+
+	// Target GCS URL (eg. gs://<bucket-name>/<path>/<sub-path>)
+	target: string @dagger(input)
+
+	// Delete files that already exist on remote destination
+	delete: *false | true @dagger(input)
+
+	// Object content type
+	contentType: string | *"" @dagger(input)
+
+	// Always write the object to GCS
+	always: *true | false @dagger(input)
+
+	// URL of the uploaded GCS object
+	url: {
+		string
+
+		#up: [
+			op.#Load & {
+				from: gcp.#GCloud & {
+					"config": config
+				}
+			},
+
+			op.#Exec & {
+				if always {
+					always: true
+				}
+				env: {
+					TARGET:           target
+					OPT_CONTENT_TYPE: contentType
+					if delete {
+						OPT_DELETE: "1"
+					}
+				}
+
+				mount: "/source": from: source
+
+				args: [
+					"/bin/bash",
+					"--noprofile",
+					"--norc",
+					"-eo",
+					"pipefail",
+					"-c",
+					#"""
+						opts=("-r")
+						[ -n "$OPT_CONTENT_TYPE" ] && opts+="-h Content-Type:$OPT_CONTENT_TYPE"
+						[ -n "$OPT_DELETE" ] && opts+="-d"
+						gsutil rsync ${opts[@]} /source "$TARGET"
+						echo -n "$TARGET" \
+							| sed -E 's=^gs://([^/]*)/=https://storage.cloud.google.com/\1/=' \
+							> /url
+						"""#,
+
+				]
+			},
+
+			op.#Export & {
+				source: "/url"
+				format: "string"
+			},
+		]
+	} @dagger(output)
+}
--- a/stdlib/gcp/gcs/tests/gcs.cue
+++ b/stdlib/gcp/gcs/tests/gcs.cue
@ -0,0 +1,43 @@
+package gcs
+
+import (
+	"alpha.dagger.io/dagger"
+	"alpha.dagger.io/gcp"
+	"alpha.dagger.io/random"
+)
+
+TestConfig: {
+	gcpConfig: gcp.#Config
+	bucket:    string @dagger(input)
+}
+
+TestDirectory: dagger.#Artifact
+
+TestGCSObject: {
+	suffix: random.#String & {
+		seed: "gcs"
+	}
+
+	target: "gs://\(TestConfig.bucket)/\(suffix.out)/"
+
+	deploy: #Object & {
+		always:   true
+		config:   TestConfig.gcpConfig
+		source:   TestDirectory
+		"target": target
+	}
+
+	verifyFile: #VerifyGCS & {
+		config: TestConfig.gcpConfig
+		target: deploy.target
+		url:    deploy.url
+		file:   "dirFile.txt"
+	}
+
+	verifyDir: #VerifyGCS & {
+		config: TestConfig.gcpConfig
+		target: deploy.target
+		url:    deploy.url
+		file:   "foo.txt"
+	}
+}
--- a/stdlib/gcp/gcs/tests/testdata/bar/foo.txt
+++ b/stdlib/gcp/gcs/tests/testdata/bar/foo.txt
@ -0,0 +1 @@
+Test recursivity
--- a/stdlib/gcp/gcs/tests/testdata/dirFile.txt
+++ b/stdlib/gcp/gcs/tests/testdata/dirFile.txt
@ -0,0 +1 @@
+Test directory
--- a/stdlib/gcp/gcs/tests/verify.cue
+++ b/stdlib/gcp/gcs/tests/verify.cue
@ -0,0 +1,89 @@
+package gcs
+
+import (
+	"alpha.dagger.io/alpine"
+	"alpha.dagger.io/dagger/op"
+	"alpha.dagger.io/gcp"
+)
+
+#List: {
+	// GCP Config
+	config: gcp.#Config
+
+	// Target GCP URL (e.g. gs://<bucket-name>/<path>/<sub-path>)
+	target: string
+
+	// URL: dummy URL, used to force a dependency
+	url: string
+
+	contents: {
+		string
+
+		#up: [
+			op.#Load & {
+				from: gcp.#GCloud & {
+					"config": config
+				}
+			},
+
+			op.#Exec & {
+				args: [
+					"/bin/bash",
+					"--noprofile",
+					"--norc",
+					"-eo",
+					"pipefail",
+					"-c",
+					#"""
+					gsutil ls -r \#(target) > /contents
+					"""#,
+				]
+				env: URL: url
+			},
+
+			op.#Export & {
+				source: "/contents"
+				format: "string"
+			},
+		]
+	}
+}
+
+#VerifyGCS: {
+	file:   string
+	config: gcp.#Config
+	target: string
+	url:    string
+
+	lists: #List & {
+		"config": config
+		"target": target
+		"url":    url
+	}
+
+	test: #up: [
+		op.#Load & {
+			from: alpine.#Image & {
+				package: bash: "~5.1"
+			}
+		},
+
+		op.#WriteFile & {
+			dest:    "/test"
+			content: lists.contents
+		},
+
+		op.#Exec & {
+			always: true
+			args: [
+				"/bin/bash",
+				"--noprofile",
+				"--norc",
+				"-eo",
+				"pipefail",
+				"-c",
+				"grep -q \(file) /test",
+			]
+		},
+	]
+}
--- a/stdlib/universe.bats
+++ b/stdlib/universe.bats
@ -121,6 +121,10 @@ setup() {
    dagger -e google-gcr up
 }

+@test "google cloud: gcs" {
+    dagger -e google-gcs up
+}
+
@test "google cloud: gke" {
    dagger -e google-gke up
 }