From 635b9306b896c2e3f05c296907b79a09a286988e Mon Sep 17 00:00:00 2001 From: Scott Lewis-Kelly Date: Wed, 30 Jun 2021 14:31:23 +1000 Subject: [PATCH] stdlib: Add GCS support The interface and behavior is identical to S3. Signed-off-by: Scott Lewis-Kelly --- docs/reference/universe/README.md | 1 + docs/reference/universe/gcp/gcs.md | 34 ++++++++++ stdlib/.dagger/env/google-gcs/.gitignore | 2 + stdlib/.dagger/env/google-gcs/values.yaml | 32 +++++++++ stdlib/gcp/gcloud.cue | 1 + stdlib/gcp/gcs/gcs.cue | 82 ++++++++++++++++++++++ stdlib/gcp/gcs/tests/gcs.cue | 33 +++++++++ stdlib/gcp/gcs/tests/testdata/bar/foo.txt | 1 + stdlib/gcp/gcs/tests/testdata/dirFile.txt | 1 + stdlib/gcp/gcs/tests/verify.cue | 83 +++++++++++++++++++++++ stdlib/universe.bats | 4 ++ 11 files changed, 274 insertions(+) create mode 100644 docs/reference/universe/gcp/gcs.md create mode 100644 stdlib/.dagger/env/google-gcs/.gitignore create mode 100644 stdlib/.dagger/env/google-gcs/values.yaml create mode 100644 stdlib/gcp/gcs/gcs.cue create mode 100644 stdlib/gcp/gcs/tests/gcs.cue create mode 100644 stdlib/gcp/gcs/tests/testdata/bar/foo.txt create mode 100644 stdlib/gcp/gcs/tests/testdata/dirFile.txt create mode 100644 stdlib/gcp/gcs/tests/verify.cue diff --git a/docs/reference/universe/README.md b/docs/reference/universe/README.md index 6dd66572..139b2b41 100644 --- a/docs/reference/universe/README.md +++ b/docs/reference/universe/README.md @@ -15,6 +15,7 @@ - [docker/compose](./docker/compose.md) - Docker-compose operations - [gcp](./gcp/README.md) - Google Cloud Platform - [gcp/gcr](./gcp/gcr.md) - Google Container Registry +- [gcp/gcs](./gcp/gcs.md) - Google Cloud Storage - [gcp/gke](./gcp/gke.md) - Google Kubernetes Engine - [git](./git.md) - Git operations - [go](./go.md) - Go build operations diff --git a/docs/reference/universe/gcp/gcs.md b/docs/reference/universe/gcp/gcs.md new file mode 100644 index 00000000..462d5245 --- /dev/null +++ b/docs/reference/universe/gcp/gcs.md @@ -0,0 +1,34 @@ +--- +sidebar_label: gcs +--- + +# alpha.dagger.io/gcp/gcs + +Google Cloud Storage + +```cue +import "alpha.dagger.io/gcp/gcs" +``` + +## gcs.#Object + +GCS Bucket object(s) sync + +### gcs.#Object Inputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |GCP region | +|*config.project* | `string` |GCP project | +|*config.serviceKey* | `dagger.#Secret` |GCP service key | +|*source* | `dagger.#Artifact` |Source Artifact to upload to GCS | +|*target* | `string` |Target GCS URL (eg. gs://\/\/\) | +|*delete* | `*false \| true` |Delete files that already exist on remote destination | +|*contentType* | `*"" \| string` |Object content type | +|*always* | `*true \| false` |Always write the object to GCS | + +### gcs.#Object Outputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*url* | `string` |URL of the uploaded GCS object | diff --git a/stdlib/.dagger/env/google-gcs/.gitignore b/stdlib/.dagger/env/google-gcs/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/google-gcs/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/google-gcs/values.yaml b/stdlib/.dagger/env/google-gcs/values.yaml new file mode 100644 index 00000000..b7ce2b1e --- /dev/null +++ b/stdlib/.dagger/env/google-gcs/values.yaml @@ -0,0 +1,32 @@ +plan: + module: .dagger/env/google-gcs/plan + package: ./tests +name: google-gcs +inputs: + TestConfig.gcpConfig.project: + text: dagger-ci + TestConfig.gcpConfig.region: + text: us-west2-a + TestDirectory: + dir: + path: ./gcp/gcs/tests/testdata +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1anekvutsz37ev3vceyg33p7lffulg7twfxmm5mtzeyc3yn336quqq3hq4q + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2MHRnNWlzN1haMENFRWhT + MUlUcXNvYU1DUXJzaU9jcS9TemRlZVF0LzNnCjdIVmpYM1lKQVNPRitxbXEyQzg2 + Si9qZkJZV3B4a2ZZOVE1ODFaa3NQWVUKLS0tIFRqTk5pVERHckEyYnM0NG9CelJV + dm45WXJ4NmxOUHAxeThESTBPdXg4c3cKmUZaIzSFHjwaoS+VqJBg1O9PQGNwhI9d + 9hvBy/BA0XaT5yIDjevjLP9vCAFDYl20G5Am2q939B8colWtic1Qug== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-06-29T23:52:31Z" + mac: ENC[AES256_GCM,data:mza0dLYMFGKgMV04YXtzix6QDtGR1PKPp8VQToOIAvBCqbLK8ti8TUs93t1wXl2Hn18StcUkw5SuFxSE+opn51Q/4TbfN726mnigAzWAd6nKbKLGvj4l1Bb/gDE39ENm7IxqoewMYBh1gufnqgKbGUrHFHK4ZOad7XsbF7UH0Pc=,iv:LBu0PAD6knb0SyVokjtF8daiDhQg3WTpCR1rRVIH5qY=,tag:tMpHIjr9pocxi7UIqsGvqg==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/gcp/gcloud.cue b/stdlib/gcp/gcloud.cue index bbbbd5f5..9b8831f4 100644 --- a/stdlib/gcp/gcloud.cue +++ b/stdlib/gcp/gcloud.cue @@ -28,6 +28,7 @@ import ( #""" curl -sfL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-\#(version)-linux-x86_64.tar.gz | tar -C /usr/local -zx ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/local/bin + ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/local/bin """#, ] }, diff --git a/stdlib/gcp/gcs/gcs.cue b/stdlib/gcp/gcs/gcs.cue new file mode 100644 index 00000000..cbe2f3f4 --- /dev/null +++ b/stdlib/gcp/gcs/gcs.cue @@ -0,0 +1,82 @@ +// Google Cloud Storage +package gcs + +import ( + "alpha.dagger.io/dagger" + "alpha.dagger.io/dagger/op" + "alpha.dagger.io/gcp" +) + +// GCS Bucket object(s) sync +#Object: { + + // GCP Config + config: gcp.#Config + + // Source Artifact to upload to GCS + source: dagger.#Artifact @dagger(input) + + // Target GCS URL (eg. gs:////) + target: string @dagger(input) + + // Delete files that already exist on remote destination + delete: *false | true @dagger(input) + + // Object content type + contentType: string | *"" @dagger(input) + + // Always write the object to GCS + always: *true | false @dagger(input) + + // URL of the uploaded GCS object + url: { + string + + #up: [ + op.#Load & { + from: gcp.#GCloud & { + "config": config + } + }, + + op.#Exec & { + if always { + always: true + } + env: { + TARGET: target + OPT_CONTENT_TYPE: contentType + if delete { + OPT_DELETE: "1" + } + } + + mount: "/source": from: source + + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + opts=("-r") + [ -n "$OPT_CONTENT_TYPE" ] && opts+="-h Content-Type:$OPT_CONTENT_TYPE" + [ -n "$OPT_DELETE" ] && opts+="-d" + gsutil rsync ${opts[@]} /source "$TARGET" + echo -n "$TARGET" \ + | sed -E 's=^gs://([^/]*)/=https://storage.cloud.google.com/\1/=' \ + > /url + """#, + + ] + }, + + op.#Export & { + source: "/url" + format: "string" + }, + ] + } @dagger(output) +} diff --git a/stdlib/gcp/gcs/tests/gcs.cue b/stdlib/gcp/gcs/tests/gcs.cue new file mode 100644 index 00000000..83abc367 --- /dev/null +++ b/stdlib/gcp/gcs/tests/gcs.cue @@ -0,0 +1,33 @@ +package gcs + +import ( + "alpha.dagger.io/dagger" + "alpha.dagger.io/gcp" +) + +TestConfig: gcpConfig: gcp.#Config + +bucket: "dagger-ci" + +TestDirectory: dagger.#Artifact + +TestGCSObject: { + deploy: #Object & { + always: true + config: TestConfig.gcpConfig + source: TestDirectory + target: "gs://\(bucket)/" + } + + verifyFile: #VerifyGCS & { + config: TestConfig.gcpConfig + target: deploy.target + file: "dirFile.txt" + } + + verifyDir: #VerifyGCS & { + config: TestConfig.gcpConfig + target: deploy.target + file: "foo.txt" + } +} diff --git a/stdlib/gcp/gcs/tests/testdata/bar/foo.txt b/stdlib/gcp/gcs/tests/testdata/bar/foo.txt new file mode 100644 index 00000000..616e65a3 --- /dev/null +++ b/stdlib/gcp/gcs/tests/testdata/bar/foo.txt @@ -0,0 +1 @@ +Test recursivity \ No newline at end of file diff --git a/stdlib/gcp/gcs/tests/testdata/dirFile.txt b/stdlib/gcp/gcs/tests/testdata/dirFile.txt new file mode 100644 index 00000000..590f5066 --- /dev/null +++ b/stdlib/gcp/gcs/tests/testdata/dirFile.txt @@ -0,0 +1 @@ +Test directory \ No newline at end of file diff --git a/stdlib/gcp/gcs/tests/verify.cue b/stdlib/gcp/gcs/tests/verify.cue new file mode 100644 index 00000000..57499488 --- /dev/null +++ b/stdlib/gcp/gcs/tests/verify.cue @@ -0,0 +1,83 @@ +package gcs + +import ( + "alpha.dagger.io/alpine" + "alpha.dagger.io/dagger/op" + "alpha.dagger.io/gcp" +) + +#List: { + // GCP Config + config: gcp.#Config + + // Target GCP URL (e.g. gs:////) + target?: string + + contents: { + string + + #up: [ + op.#Load & { + from: #GCloud & { + "config": config + } + }, + + op.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + gsutil ls -r \#(target) > /contents + """#, + ] + }, + + op.#Export & { + source: "/contents" + format: "string" + }, + ] + } +} + +#VerifyGCS: { + file: string + config: gcp.#Config + target: string + + lists: #List & { + "config": config + "target": target + } + + test: #up: [ + op.#Load & { + from: alpine.#Image & { + package: bash: "~5.1" + } + }, + + op.#WriteFile & { + dest: "/test" + content: lists.contents + }, + + op.#Exec & { + always: true + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + "grep -q \(file) /test", + ] + }, + ] +} diff --git a/stdlib/universe.bats b/stdlib/universe.bats index 2ff76e4a..8de32abc 100644 --- a/stdlib/universe.bats +++ b/stdlib/universe.bats @@ -121,6 +121,10 @@ setup() { dagger -e google-gcr up } +# @test "google cloud: gcs" { +# dagger -e google-gcs up +# } + @test "google cloud: gke" { dagger -e google-gke up }