diff --git a/docs/reference/universe/README.md b/docs/reference/universe/README.md index 6dd66572..139b2b41 100644 --- a/docs/reference/universe/README.md +++ b/docs/reference/universe/README.md @@ -15,6 +15,7 @@ - [docker/compose](./docker/compose.md) - Docker-compose operations - [gcp](./gcp/README.md) - Google Cloud Platform - [gcp/gcr](./gcp/gcr.md) - Google Container Registry +- [gcp/gcs](./gcp/gcs.md) - Google Cloud Storage - [gcp/gke](./gcp/gke.md) - Google Kubernetes Engine - [git](./git.md) - Git operations - [go](./go.md) - Go build operations diff --git a/docs/reference/universe/gcp/gcs.md b/docs/reference/universe/gcp/gcs.md new file mode 100644 index 00000000..462d5245 --- /dev/null +++ b/docs/reference/universe/gcp/gcs.md @@ -0,0 +1,34 @@ +--- +sidebar_label: gcs +--- + +# alpha.dagger.io/gcp/gcs + +Google Cloud Storage + +```cue +import "alpha.dagger.io/gcp/gcs" +``` + +## gcs.#Object + +GCS Bucket object(s) sync + +### gcs.#Object Inputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |GCP region | +|*config.project* | `string` |GCP project | +|*config.serviceKey* | `dagger.#Secret` |GCP service key | +|*source* | `dagger.#Artifact` |Source Artifact to upload to GCS | +|*target* | `string` |Target GCS URL (eg. gs://\/\/\) | +|*delete* | `*false \| true` |Delete files that already exist on remote destination | +|*contentType* | `*"" \| string` |Object content type | +|*always* | `*true \| false` |Always write the object to GCS | + +### gcs.#Object Outputs + +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*url* | `string` |URL of the uploaded GCS object | diff --git a/stdlib/.dagger/env/google-gcs/.gitignore b/stdlib/.dagger/env/google-gcs/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/google-gcs/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/google-gcs/values.yaml b/stdlib/.dagger/env/google-gcs/values.yaml new file mode 100644 index 00000000..298a1223 --- /dev/null +++ b/stdlib/.dagger/env/google-gcs/values.yaml @@ -0,0 +1,36 @@ +plan: + module: ./gcp/gcs/ + package: ./tests +name: google-gcs +inputs: + TestConfig.bucket: + text: dagger-ci + TestConfig.gcpConfig.project: + text: dagger-ci + TestConfig.gcpConfig.region: + text: us-west2-a + TestConfig.gcpConfig.serviceKey: + secret: ENC[AES256_GCM,data:UEKTXvyrBgHKOYE9vSGoHua9wWALjghxWu+ui9K3MAS+1mnVlc1qjTbwv/1/hIIkRNlyhY6WlN0k3x2imusFFInzrNZ5G4FJHGiP/zaazd7shUS8LZsh1cL0I1jnsaDJaz4Zw0yVu+FT1z2/+9l81U9MrtvbLNKFSqZJsrymZl5lUCxiRsUEBiC0/rOoOlQ88kfnxUdBXnG7ABciqPUK7cYaMo5RbB1a9YfacB8S2sosClxK727jUgD20I12ru+y5Y/hg00BhBl9bIg35VTI8PFeZvRqQowqaJO+i1BjQbBYef8s9faYdZGEP0hUrvUpPek4Z0ZBDRbxRfRFAiXBbWbvCJErGlsmcNGqllixgDbcYQNKBTjZEKke4PGHtahmPXOIOO9/fxnUCoci//azJ5fUP0Kdiw06DQYQnngwRSA/nOqTBiuWcfiTLY00iQxoh+8Mt5/CMTXhRz8PpRpvXtKLe4ogaVbEBMOXe8+nMtwt0H6kV+YmHgA+vVHw0WYtBF6gmKnXuGWT/dz94SHGe9oMiK7H7KtfRCyN1SBAx3H9R3qitt9114TlWRBvfrVtDsb/E0MQdnbXZQirJ/2ev78DTc5bdRTFus2vvHZB2RP+wr7sB8A+jcWS1RBv0NJOZoPlHqKIfB93NBhI8wU+lqEbTn2Zm0LAvGVypws3ci+0GJFReIok9yGc/WucFBDMWi8tDOwUdbreX7EpdGLSn9Rwq0Oeuo5Udy5Qo4xFAC/v2yCzqb7MrZzX1S+OHyNObUTGuEyTOL8t91KYBebHtLK0Ud9qQoYuO5IvYCGeKIHr5FDFCYKDJAlW8Y9Iw27LsU45rqeHTjUi1BqOLfXZbns79w8WYiz5bqc3eMXmYOgCKtU5BwLo9LV7WR+FW8cCi9gSJHnRbc8uT9V1NWaGeoKuaPXpBWLH/GTKXEKuy1HXz+yPgQsYYN2yrGNgmgXpKbxgp92h6LRA/PRjxrjkAV8kVdivxG3GUFVfq4ZtGlLdRrcYCkDIZVpwPrNp8Qg5NLn5xhWj1hsqqOOJdeDHHLkS48A9pLAhPubRWH0s1IixLstnJ95DGIZDE0QpEh7STctqvS7SVhclroKI5xzT7w3C78fxTKC//JkdrV0/rQ2CQ1u+emaCUt3X1WwYofudoMjp5iESscDvw6PnqzubUT51FwGtAf2hkT07VQWZ2CcNP4dvYgDSsc26Y0hwWdr+ppCy1xbmpENU5moUPELtF9w2da1zG5N32Fcxr9VOqOGwa4pqkasIQPCRdqsEa/34g95L7Z1BAXpMxCLTbW5nxD3wQBsrsTjO0rmBh4n9BdKjqj5GrXOTceAaJ+L3o07aVIzICJ5HGo7GQXjaQOABzysbGr7bdBIQyp//ruqXNLVSaylIunZJY4/HFdqetZj6jfb4rE+/GkjFLw23Ym8RIDYJQsTAR8OdwZNtVPrILdve1ohlyBW/nS+gy5dZ3qHnE9ZxbgN6FpRYr9G4acK3yCyMeuBdiB6Tqz0o6xz/c2WLWpWcw/HivVce53MfbqMhj0K/jjpl+fsJT/+ZkaPviDrOtKR/WPz/ewNwQ7pZgCubtlzCmPw8pBKjP+uI9tvwydgnV8I/LdEvS0IrP9m22kPZiw/m7mPQPkDA7IzQSIlrmWMCt/iJ0rvVEtkyi1pPyzVVby4OyzvyhexVcFu2X1x5jOqfUL3eEjO+IhMJufVCOVlUKNHPci0r4KQZ07n1wPCjUoCN4MMl9D8yaP0l8SQmSmp9/DFVZ1Cd5m+O0cmaQfZX8dYV52uxkPnkRNn9SnKCz5Nf8tBG8p+kvznEYw6d4UKQLocFdzeBU6FBa4zQSf+oXHghds9YGJoWNAGzD9ucgVNyu4hU2rq7UDnAwT/pm8FTbw/OM2uCyPr80J3CpgKPdA0+jEvBNxlezsZeLyTntoZLIstgllETxC9QjqfCVtnQYBYT8oFCXZPadyBZPPVrXvQORCpRXJCQ9iyoiKTQCCOGqGyIgC+23Q0ebmZPr7e4I5JbBZWeh6ek2RRWOZMI0lkLeE72KO+9h8HPxmgMhPtDes/l6DmWjAPb6IwZSM9SseTYF447HL5rQKkv4x7W4Cu1pACSc/2qqdUwnC6jydk8EY2zPa9kdguwdSiDr+KwASZex2MKMJa+peXhTDLqB7jxiO7snU4r0nk7E0B3IBqWM89PVskyNTJXqTkAXyzHWag1S8URPhk9Q9UYpyVCWRzs4nfXAX2WgCJl9iw8ZXrIJZ3fGs3vXQMYI/kGJ7rGIDWhcQEDJGKJYrBCh6zNsbSzBagd5AQ6DNIUEvTUcPvxZP6sy+uiiMMqErDgbi2IT3G6E5sEL0sO7GtrVWoxDQWXX80vtf146fnNNp4uS2wGta7b4kguAt39olceX7nUrYgzLm1aXcr0ObbRjcPdeSlNGLTJGlZRNtvTlFdT1RgTQBfRdG3d1JCraCGsAWxBC+Bwa/zAcdIwUNnMfgdZ5eOAyiBTmoGAAOkUr7avSVvP5Xyrk9JlNzeArrjZUFMYOcXBfkMquqD1NoPHUZ12u9jCoj109zJquxcJ1zGT5TylJYSTc9xz46TfLrz5KvsMhfetBx0ocY8lXjbs3pzB6ZswDGzF4j0eiWBfu9UZoiQVtt1eIfAPoZO9u/n2Wbpx+ov+mJFf0dZklKDED5CUbmj8/izOdE9KbiPDFr1nZZQbO+YwJYhjxxqxzwC41CvH5XQTCCadeWgb5cHEwwunFRVCYeEFvTlP6nR9pS19UZoQvPNk5NEnZVKxWVD1lK18frvX8MxIpB2xaSbUkvGi3tueinLa7OYVfjirBJ1GYbuADa4H0KOxi4MP59ah6TOmsZstA6plFbtRRtk4Iofzu1mAyUN52SZzwCvH5/IxOh6vQxg7EcMfB8O5+S87PeE3vd1ch+UVDwQ3fvNmjd7/77XdKZiR7bwiDxodxABpG4iYrxcfkGPOTCKUlisgFNNmSmwG7FJMYWkz3RcEHib/biklnEKbm/9G5000cwZXXglZb8QjleSklkSEZa4DYec1SNvD/KwRbk5lYr03TDl7DrZo3WjKWvWGMUs3,iv:tPbOGayR7NiXcuHWjX0pX/nSitOxmsr4qqrc6irlIJI=,tag:apejA4UTYTuwT4CUSeoaRQ==,type:str] + TestDirectory: + dir: + path: ./gcp/gcs/tests/testdata +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCRThSTEE1Rm5HU2Y2NkJZ + SEd2blAyeEVnMHBqRGxXMEQ3TGFzWTBwd1EwCnA0OFVmTCsxSmpNV29adGt2ZHFH + WE9vN1ZoNENFV2t1TGVuZkdwVndNbVUKLS0tIGpHZEptYWxEZVNjcXF4NkoyWHRv + ZXd6Qmd1YUtxMnVTVkYybWgrV3pVK2MKowMeOZU3j3BxERT0DwhQYCGUDBK6gCdo + WByubiBATdsb7h7ytCC4HutWppynK4MpU+Ya9NP83AZuXo+Wa2u6aQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-06-30T10:44:08Z" + mac: ENC[AES256_GCM,data:bVjVtONv7ILxZ3UkYy/l3MnuJB8g4EG++fBfeL/CQPHyembwBURC3nxfB0IdqQmPsa0/ahycNawIvB/owy+Sw6yyaDWGtd/8Jn0CILLZ4Nyx9lNN95J4Ad0FbUKr24Fu17I339rB1GrEjAaTs1qcle53sp4Wq9Blg26Vkut9jmc=,iv:TdFDptBZrScOfywWsIhoLijSFQL18DF94bKMh5DfW5s=,tag:wVoa1K/Nyx05nu+Op0Viuw==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/gcp/gcloud.cue b/stdlib/gcp/gcloud.cue index bbbbd5f5..9b8831f4 100644 --- a/stdlib/gcp/gcloud.cue +++ b/stdlib/gcp/gcloud.cue @@ -28,6 +28,7 @@ import ( #""" curl -sfL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-\#(version)-linux-x86_64.tar.gz | tar -C /usr/local -zx ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/local/bin + ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/local/bin """#, ] }, diff --git a/stdlib/gcp/gcs/gcs.cue b/stdlib/gcp/gcs/gcs.cue new file mode 100644 index 00000000..cbe2f3f4 --- /dev/null +++ b/stdlib/gcp/gcs/gcs.cue @@ -0,0 +1,82 @@ +// Google Cloud Storage +package gcs + +import ( + "alpha.dagger.io/dagger" + "alpha.dagger.io/dagger/op" + "alpha.dagger.io/gcp" +) + +// GCS Bucket object(s) sync +#Object: { + + // GCP Config + config: gcp.#Config + + // Source Artifact to upload to GCS + source: dagger.#Artifact @dagger(input) + + // Target GCS URL (eg. gs:////) + target: string @dagger(input) + + // Delete files that already exist on remote destination + delete: *false | true @dagger(input) + + // Object content type + contentType: string | *"" @dagger(input) + + // Always write the object to GCS + always: *true | false @dagger(input) + + // URL of the uploaded GCS object + url: { + string + + #up: [ + op.#Load & { + from: gcp.#GCloud & { + "config": config + } + }, + + op.#Exec & { + if always { + always: true + } + env: { + TARGET: target + OPT_CONTENT_TYPE: contentType + if delete { + OPT_DELETE: "1" + } + } + + mount: "/source": from: source + + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + opts=("-r") + [ -n "$OPT_CONTENT_TYPE" ] && opts+="-h Content-Type:$OPT_CONTENT_TYPE" + [ -n "$OPT_DELETE" ] && opts+="-d" + gsutil rsync ${opts[@]} /source "$TARGET" + echo -n "$TARGET" \ + | sed -E 's=^gs://([^/]*)/=https://storage.cloud.google.com/\1/=' \ + > /url + """#, + + ] + }, + + op.#Export & { + source: "/url" + format: "string" + }, + ] + } @dagger(output) +} diff --git a/stdlib/gcp/gcs/tests/gcs.cue b/stdlib/gcp/gcs/tests/gcs.cue new file mode 100644 index 00000000..22530ce2 --- /dev/null +++ b/stdlib/gcp/gcs/tests/gcs.cue @@ -0,0 +1,43 @@ +package gcs + +import ( + "alpha.dagger.io/dagger" + "alpha.dagger.io/gcp" + "alpha.dagger.io/random" +) + +TestConfig: { + gcpConfig: gcp.#Config + bucket: string @dagger(input) +} + +TestDirectory: dagger.#Artifact + +TestGCSObject: { + suffix: random.#String & { + seed: "gcs" + } + + target: "gs://\(TestConfig.bucket)/\(suffix.out)/" + + deploy: #Object & { + always: true + config: TestConfig.gcpConfig + source: TestDirectory + "target": target + } + + verifyFile: #VerifyGCS & { + config: TestConfig.gcpConfig + target: deploy.target + url: deploy.url + file: "dirFile.txt" + } + + verifyDir: #VerifyGCS & { + config: TestConfig.gcpConfig + target: deploy.target + url: deploy.url + file: "foo.txt" + } +} diff --git a/stdlib/gcp/gcs/tests/testdata/bar/foo.txt b/stdlib/gcp/gcs/tests/testdata/bar/foo.txt new file mode 100644 index 00000000..616e65a3 --- /dev/null +++ b/stdlib/gcp/gcs/tests/testdata/bar/foo.txt @@ -0,0 +1 @@ +Test recursivity \ No newline at end of file diff --git a/stdlib/gcp/gcs/tests/testdata/dirFile.txt b/stdlib/gcp/gcs/tests/testdata/dirFile.txt new file mode 100644 index 00000000..590f5066 --- /dev/null +++ b/stdlib/gcp/gcs/tests/testdata/dirFile.txt @@ -0,0 +1 @@ +Test directory \ No newline at end of file diff --git a/stdlib/gcp/gcs/tests/verify.cue b/stdlib/gcp/gcs/tests/verify.cue new file mode 100644 index 00000000..4ad06e84 --- /dev/null +++ b/stdlib/gcp/gcs/tests/verify.cue @@ -0,0 +1,89 @@ +package gcs + +import ( + "alpha.dagger.io/alpine" + "alpha.dagger.io/dagger/op" + "alpha.dagger.io/gcp" +) + +#List: { + // GCP Config + config: gcp.#Config + + // Target GCP URL (e.g. gs:////) + target: string + + // URL: dummy URL, used to force a dependency + url: string + + contents: { + string + + #up: [ + op.#Load & { + from: gcp.#GCloud & { + "config": config + } + }, + + op.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + gsutil ls -r \#(target) > /contents + """#, + ] + env: URL: url + }, + + op.#Export & { + source: "/contents" + format: "string" + }, + ] + } +} + +#VerifyGCS: { + file: string + config: gcp.#Config + target: string + url: string + + lists: #List & { + "config": config + "target": target + "url": url + } + + test: #up: [ + op.#Load & { + from: alpine.#Image & { + package: bash: "~5.1" + } + }, + + op.#WriteFile & { + dest: "/test" + content: lists.contents + }, + + op.#Exec & { + always: true + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + "grep -q \(file) /test", + ] + }, + ] +} diff --git a/stdlib/universe.bats b/stdlib/universe.bats index 2ff76e4a..07c9ecbf 100644 --- a/stdlib/universe.bats +++ b/stdlib/universe.bats @@ -121,6 +121,10 @@ setup() { dagger -e google-gcr up } +@test "google cloud: gcs" { + dagger -e google-gcs up +} + @test "google cloud: gke" { dagger -e google-gke up }