From 40d4c95bffaf85dce12658f46b5e6f3358796a1d Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 26 May 2021 18:03:48 -0700 Subject: [PATCH] aws: use secrets Signed-off-by: Andrea Luzzardi --- stdlib/aws/aws.cue | 105 ++++-------------- stdlib/aws/ecr/ecr.cue | 40 +++++-- stdlib/aws/s3/s3.cue | 94 ++++++++++------ stdlib/netlify/netlify.cue | 2 +- stdlib/netlify/netlify.sh.cue | 2 +- tests/stdlib.bats | 8 +- .../aws/ecr/.dagger/env/default/.gitignore | 2 + .../{ => .dagger/env/default/plan}/ecr.cue | 0 .../{ => .dagger/env/default/plan}/random.cue | 0 .../aws/ecr/.dagger/env/default/values.yaml | 26 +++++ .../aws/s3/.dagger/env/default/.gitignore | 2 + .../s3/{ => .dagger/env/default/plan}/s3.cue | 6 + .../{ => .dagger/env/default/plan}/verify.cue | 56 ++++++---- .../aws/s3/.dagger/env/default/values.yaml | 30 +++++ 14 files changed, 219 insertions(+), 154 deletions(-) create mode 100644 tests/stdlib/aws/ecr/.dagger/env/default/.gitignore rename tests/stdlib/aws/ecr/{ => .dagger/env/default/plan}/ecr.cue (100%) rename tests/stdlib/aws/ecr/{ => .dagger/env/default/plan}/random.cue (100%) create mode 100644 tests/stdlib/aws/ecr/.dagger/env/default/values.yaml create mode 100644 tests/stdlib/aws/s3/.dagger/env/default/.gitignore rename tests/stdlib/aws/s3/{ => .dagger/env/default/plan}/s3.cue (79%) rename tests/stdlib/aws/s3/{ => .dagger/env/default/plan}/verify.cue (51%) create mode 100644 tests/stdlib/aws/s3/.dagger/env/default/values.yaml diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index c8e8d0f4..23ef9040 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -18,6 +18,7 @@ import ( // Re-usable aws-cli component #CLI: { + config: #Config package: [string]: string | bool #up: [ @@ -30,86 +31,26 @@ import ( "package": "aws-cli": "=~1.18" } }, + op.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" + aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" + + aws configure set default.region "$AWS_DEFAULT_REGION" + aws configure set default.cli_pager "" + aws configure set default.output "json" + """# + ] + mount: "/run/secrets/access_key": secret: config.accessKey + mount: "/run/secrets/secret_key": secret: config.secretKey + env: AWS_DEFAULT_REGION: config.region + }, ] -} - -// Helper for writing scripts based on AWS CLI -#Script: { - // AWS code - config: #Config - - // Script code (bash) - code: string - - // Extra pkgs to install - package: [string]: string | bool - - // Files to mount - files: [string]: string - - // Env variables - env: [string]: string - - // Export file - export: string - - // Always execute the script? - always?: bool - - // Directory - dir?: dagger.#Artifact - - out: { - string - - #up: [ - op.#Load & { - from: #CLI & { - "package": package - } - }, - op.#Mkdir & { - path: "/inputs" - }, - for k, v in files { - op.#WriteFile & { - dest: k - content: v - } - }, - op.#WriteFile & { - dest: "/entrypoint.sh" - content: code - }, - op.#Exec & { - if always != _|_ { - "always": always - } - args: [ - "/bin/bash", - "--noprofile", - "--norc", - "-eo", - "pipefail", - "/entrypoint.sh", - ] - "env": env - "env": { - AWS_ACCESS_KEY_ID: config.accessKey - AWS_SECRET_ACCESS_KEY: config.secretKey - AWS_DEFAULT_REGION: config.region - AWS_REGION: config.region - AWS_DEFAULT_OUTPUT: "json" - AWS_PAGER: "" - } - if dir != _|_ { - mount: "/inputs/source": from: dir - } - }, - op.#Export & { - source: export - format: "string" - }, - ] - } -} +} \ No newline at end of file diff --git a/stdlib/aws/ecr/ecr.cue b/stdlib/aws/ecr/ecr.cue index 8ffdf038..6a7018d8 100644 --- a/stdlib/aws/ecr/ecr.cue +++ b/stdlib/aws/ecr/ecr.cue @@ -2,6 +2,7 @@ package ecr import ( "dagger.io/dagger" + "dagger.io/dagger/op" "dagger.io/aws" ) @@ -15,14 +16,37 @@ import ( // ECR credentials username: "AWS" - secret: out @dagger(output) + secret: { + @dagger(output) + string - aws.#Script & { - always: true - "config": config - export: "/out" - code: """ - aws ecr get-login-password > /out - """ + #up: [ + op.#Load & { + from: aws.#CLI & { + "config": config + } + }, + + op.#Exec & { + always: true + + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + aws ecr get-login-password > /out + """# + ] + }, + + op.#Export & { + source: "/out" + format: "string" + } + ] } } diff --git a/stdlib/aws/s3/s3.cue b/stdlib/aws/s3/s3.cue index 3c380979..1d33c752 100644 --- a/stdlib/aws/s3/s3.cue +++ b/stdlib/aws/s3/s3.cue @@ -2,6 +2,7 @@ package s3 import ( "dagger.io/dagger" + "dagger.io/dagger/op" "dagger.io/aws" ) @@ -23,46 +24,69 @@ import ( // Object content type contentType: string | *"" @dagger(input) - // URL of the uploaded S3 object - url: out @dagger(output) - // Always write the object to S3 always?: bool @dagger(input) - out: string - aws.#Script & { - if always != _|_ { - "always": always - } - files: { + // URL of the uploaded S3 object + url: { + @dagger(output) + string + + #up: [ + op.#Load & { + from: aws.#CLI & { + "config": config + } + }, + if sourceInline != _|_ { - "/inputs/source": sourceInline + op.#WriteFile & { + dest: "/source" + content: sourceInline + } } - "/inputs/target": target - if contentType != "" { - "/inputs/content_type": contentType + + op.#Exec & { + if always != _|_ { + "always": always + } + env: { + TARGET: target + CONTENT_TYPE: contentType + } + + if sourceInline == _|_ { + mount: "/source": from: source + } + + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + opts="" + op=cp + if [ -d /source ]; then + op=sync + fi + if [ -n "$CONTENT_TYPE" ]; then + opts="--content-type $CONTENT_TYPE" + fi + aws s3 $op $opts /source "$TARGET" + echo "$TARGET" \ + | sed -E 's=^s3://([^/]*)/=https://\1.s3.amazonaws.com/=' \ + > /url + """# + ] + }, + + op.#Export & { + source: "/url" + format: "string" } - } - - export: "/url" - - code: #""" - opts="" - op=cp - if [ -d /inputs/source ]; then - op=sync - fi - if [ -f /inputs/content_type ]; then - opts="--content-type $(cat /inputs/content_type)" - fi - aws s3 $op $opts /inputs/source "$(cat /inputs/target)" - cat /inputs/target \ - | sed -E 's=^s3://([^/]*)/=https://\1.s3.amazonaws.com/=' \ - > /url - """# - - if sourceInline == _|_ { - dir: source - } + ] } } diff --git a/stdlib/netlify/netlify.cue b/stdlib/netlify/netlify.cue index 79fabdcc..44b44778 100644 --- a/stdlib/netlify/netlify.cue +++ b/stdlib/netlify/netlify.cue @@ -84,6 +84,6 @@ import ( } dir: "/src" mount: "/src": from: contents - mount: "/token": secret: account.token + mount: "/run/secrets/token": secret: account.token } } diff --git a/stdlib/netlify/netlify.sh.cue b/stdlib/netlify/netlify.sh.cue index 27667a9e..6feb87ab 100644 --- a/stdlib/netlify/netlify.sh.cue +++ b/stdlib/netlify/netlify.sh.cue @@ -1,7 +1,7 @@ package netlify #Site: ctr: command: #""" - export NETLIFY_AUTH_TOKEN="$(cat /token)" + export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)" create_site() { url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" diff --git a/tests/stdlib.bats b/tests/stdlib.bats index 409169f1..cf4325f7 100644 --- a/tests/stdlib.bats +++ b/tests/stdlib.bats @@ -43,9 +43,7 @@ setup() { } @test "stdlib: aws: s3" { - skip_unless_secrets_available "$TESTDIR"/stdlib/aws/inputs.yaml - - "$DAGGER" compute "$TESTDIR"/stdlib/aws/s3 --input-dir TestDirectory="$TESTDIR"/stdlib/aws/s3/testdata --input-yaml "$TESTDIR"/stdlib/aws/inputs.yaml + "$DAGGER" up -w "$TESTDIR"/stdlib/aws/s3 } @test "stdlib: aws: eks" { @@ -55,9 +53,7 @@ setup() { } @test "stdlib: aws: ecr" { - skip_unless_secrets_available "$TESTDIR"/stdlib/aws/inputs.yaml - - "$DAGGER" compute "$TESTDIR"/stdlib/aws/ecr --input-yaml "$TESTDIR"/stdlib/aws/inputs.yaml + "$DAGGER" up -w "$TESTDIR"/stdlib/aws/ecr } @test "stdlib: gcp: gke" { diff --git a/tests/stdlib/aws/ecr/.dagger/env/default/.gitignore b/tests/stdlib/aws/ecr/.dagger/env/default/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/tests/stdlib/aws/ecr/.dagger/env/default/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/tests/stdlib/aws/ecr/ecr.cue b/tests/stdlib/aws/ecr/.dagger/env/default/plan/ecr.cue similarity index 100% rename from tests/stdlib/aws/ecr/ecr.cue rename to tests/stdlib/aws/ecr/.dagger/env/default/plan/ecr.cue diff --git a/tests/stdlib/aws/ecr/random.cue b/tests/stdlib/aws/ecr/.dagger/env/default/plan/random.cue similarity index 100% rename from tests/stdlib/aws/ecr/random.cue rename to tests/stdlib/aws/ecr/.dagger/env/default/plan/random.cue diff --git a/tests/stdlib/aws/ecr/.dagger/env/default/values.yaml b/tests/stdlib/aws/ecr/.dagger/env/default/values.yaml new file mode 100644 index 00000000..b8dcf96d --- /dev/null +++ b/tests/stdlib/aws/ecr/.dagger/env/default/values.yaml @@ -0,0 +1,26 @@ +name: default +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:iu6LfQNgGZUVnHVeMRYPrcBtlZk=,iv:U5PLxDKXwJnUDdk1ayFGvvJfWdVqh1PK5ujb20YYPP0=,tag:QyqIJRiR6nE16ZDV0CP7Pw==,type:str] + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:Q/W+KH3NEouGt6C5S+KiC43837soYi2Mjb/z5K8rD9gtaNaBjjkJHg==,iv:8nGEzLXd91rF5YBZ/EdQoMN27yrpc0sgm26DEvIuSHM=,tag:/oyKl/vj5MJAm+jZMOOAuQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-05-27T00:53:54Z" + mac: ENC[AES256_GCM,data:ho/e/xUzRdwwe3VRCz9p8UNHgxdhAxkNtWUJLS5fEXBGnw28hjwNBbPYN78bX0k9SQ/5bgvXT2O/Z+zmOSWfrCYD2eojh9mDR4aCV5m/liVh5Dxha65u6zPl9VVcSunYg3wqe9Zl+pMG8BJXvczQS7S5QEGEaWojfaA/o7HM1BE=,iv:o/cVw6GBCCdgIqIZGDzqSCiBHUmrhAoIRcyGS9P83j0=,tag:WSQO0C0lPH2vOzl07rmRGg==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/tests/stdlib/aws/s3/.dagger/env/default/.gitignore b/tests/stdlib/aws/s3/.dagger/env/default/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/tests/stdlib/aws/s3/.dagger/env/default/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/tests/stdlib/aws/s3/s3.cue b/tests/stdlib/aws/s3/.dagger/env/default/plan/s3.cue similarity index 79% rename from tests/stdlib/aws/s3/s3.cue rename to tests/stdlib/aws/s3/.dagger/env/default/plan/s3.cue index 1949c1db..fab14d50 100644 --- a/tests/stdlib/aws/s3/s3.cue +++ b/tests/stdlib/aws/s3/.dagger/env/default/plan/s3.cue @@ -22,6 +22,8 @@ TestS3UploadFile: { } verify: #VerifyS3 & { + config: TestConfig.awsConfig + target: deploy.target file: "test.txt" } } @@ -36,10 +38,14 @@ TestS3UploadDir: { } verifyFile: #VerifyS3 & { + config: TestConfig.awsConfig + target: deploy.target file: "dirFile.txt" } verifyDir: #VerifyS3 & { + config: TestConfig.awsConfig + target: deploy.target file: "foo.txt" } } diff --git a/tests/stdlib/aws/s3/verify.cue b/tests/stdlib/aws/s3/.dagger/env/default/plan/verify.cue similarity index 51% rename from tests/stdlib/aws/s3/verify.cue rename to tests/stdlib/aws/s3/.dagger/env/default/plan/verify.cue index b84380d3..92724885 100644 --- a/tests/stdlib/aws/s3/verify.cue +++ b/tests/stdlib/aws/s3/.dagger/env/default/plan/verify.cue @@ -13,30 +13,48 @@ import ( // Target S3 URL (e.g. s3:////) target?: string - // Export folder - export: "/contents" + contents: { + string - // Script - aws.#Script & { - code: """ - aws s3 ls --recursive \(target) >> /contents - """ + #up: [ + op.#Load & { + from: aws.#CLI & { + "config": config + } + }, + + op.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + #""" + aws s3 ls --recursive \#(target) > /contents + """# + ] + }, + + op.#Export & { + source: "/contents" + format: "string" + } + ] } } #VerifyS3: { file: string + config: aws.#Config + target: string lists: #List & { - config: TestConfig.awsConfig - target: "s3://\(bucket)" + "config": config + "target": target } - #CheckFiles: - """ - grep -q \(file) /test - """ - test: #up: [ op.#Load & { from: alpine.#Image & { @@ -46,12 +64,7 @@ import ( op.#WriteFile & { dest: "/test" - content: lists.out - }, - - op.#WriteFile & { - dest: "/checkFiles.sh" - content: #CheckFiles + content: lists.contents }, op.#Exec & { @@ -62,7 +75,8 @@ import ( "--norc", "-eo", "pipefail", - "/checkFiles.sh", + "-c", + "grep -q \(file) /test" ] }, ] diff --git a/tests/stdlib/aws/s3/.dagger/env/default/values.yaml b/tests/stdlib/aws/s3/.dagger/env/default/values.yaml new file mode 100644 index 00000000..dbf764c7 --- /dev/null +++ b/tests/stdlib/aws/s3/.dagger/env/default/values.yaml @@ -0,0 +1,30 @@ +name: default +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:iu6LfQNgGZUVnHVeMRYPrcBtlZk=,iv:U5PLxDKXwJnUDdk1ayFGvvJfWdVqh1PK5ujb20YYPP0=,tag:QyqIJRiR6nE16ZDV0CP7Pw==,type:str] + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:Q/W+KH3NEouGt6C5S+KiC43837soYi2Mjb/z5K8rD9gtaNaBjjkJHg==,iv:8nGEzLXd91rF5YBZ/EdQoMN27yrpc0sgm26DEvIuSHM=,tag:/oyKl/vj5MJAm+jZMOOAuQ==,type:str] + TestDirectory: + dir: + path: ./testdata + include: [] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-05-27T00:13:13Z" + mac: ENC[AES256_GCM,data:uqGhc0e6mQp5kdKvJTFz+yjcc5WUtLBcsqkzh0NeJhP9nztpX1TJfqBeyGfd7pwltL6b9YXLdJx/myCMxvJ6O8bS726AxE4ogcRgUGP6d5Q5aXw9i7VkLgVKY+gJZCbT+r80RiMqm23x3CPAPNjEsPh5nfgdNsN5ltJmq7IUGj0=,iv:Mw56hEghRGw6tLP6rhe78yD/blqgX2roeQRDiJ6+kAI=,tag:qE2LtSZPxDhITtdnsvrYfA==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1