From d5aa68fe2bc7481ff5aa617b28a8392258bdbb25 Mon Sep 17 00:00:00 2001 From: Tom Chauveau Date: Thu, 22 Apr 2021 19:14:17 +0200 Subject: [PATCH 1/3] Add test on AWS s3 Signed-off-by: Tom Chauveau --- tests/stdlib.bats | 6 +++ tests/stdlib/aws/s3/s3.cue | 24 ++++++++++++ tests/stdlib/aws/s3/verify.cue | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 tests/stdlib/aws/s3/s3.cue create mode 100644 tests/stdlib/aws/s3/verify.cue diff --git a/tests/stdlib.bats b/tests/stdlib.bats index 4f71b059..0d7c2aca 100644 --- a/tests/stdlib.bats +++ b/tests/stdlib.bats @@ -37,3 +37,9 @@ setup() { "$DAGGER" compute "$TESTDIR"/stdlib/kubernetes/helm --input-dir kubeconfig=~/.kube --input-dir TestHelmSimpleChart.deploy.chartSource="$TESTDIR"/stdlib/kubernetes/helm/testdata/mychart } + +@test "stdlib: s3" { + skip_unless_secrets_available "$TESTDIR"/stdlib/aws/inputs.yaml + + "$DAGGER" compute "$TESTDIR"/stdlib/aws/s3 --input-yaml "$TESTDIR"/stdlib/aws/inputs.yaml +} \ No newline at end of file diff --git a/tests/stdlib/aws/s3/s3.cue b/tests/stdlib/aws/s3/s3.cue new file mode 100644 index 00000000..92786264 --- /dev/null +++ b/tests/stdlib/aws/s3/s3.cue @@ -0,0 +1,24 @@ +package s3 + +import ( + "dagger.io/aws" + "dagger.io/aws/s3" +) + +TestConfig: awsConfig: aws.#Config & { + region: "us-east-2" +} + +bucket: "dagger-ci" + +content: "A simple test sentence" + +TestS3UploadFile: { + deploy: s3.#Put & { + config: TestConfig.awsConfig + sourceInline: content + target: "s3://\(bucket)/test.txt" + } + + verify: #VerifyS3 +} diff --git a/tests/stdlib/aws/s3/verify.cue b/tests/stdlib/aws/s3/verify.cue new file mode 100644 index 00000000..90befbcb --- /dev/null +++ b/tests/stdlib/aws/s3/verify.cue @@ -0,0 +1,67 @@ +package s3 + +import ( + "dagger.io/aws" + "dagger.io/alpine" + "dagger.io/dagger/op" +) + +#List: { + // AWS Config + config: aws.#Config + + // Target S3 URL (e.g. s3:////) + target?: string + + // Export folder + export: "/contents" + + // Script + aws.#Script & { + code: """ + aws s3 ls \(target) > /contents + """ + } +} + +#VerifyS3: { + lists: #List & { + config: TestConfig.awsConfig + target: "s3://\(bucket)" + } + + #CheckFiles: + """ + grep -q test.txt /test + """ + + test: #up: [ + op.#Load & { + from: alpine.#Image & { + package: bash: "~5.1" + } + }, + + op.#WriteFile & { + dest: "/test" + content: lists.out + }, + + op.#WriteFile & { + dest: "/checkFiles.sh" + content: #CheckFiles + }, + + op.#Exec & { + always: true + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "/checkFiles.sh", + ] + }, + ] +} From 226be80d566e4275ebb1c8eb0265714baf15f730 Mon Sep 17 00:00:00 2001 From: Tom Chauveau Date: Thu, 22 Apr 2021 20:56:06 +0200 Subject: [PATCH 2/3] Fix directory upload on AWS s3 Signed-off-by: Tom Chauveau --- stdlib/aws/aws.cue | 6 ++++++ stdlib/aws/s3/s3.cue | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index 79c3b64f..975454cb 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -56,6 +56,9 @@ import ( // Always execute the script? always?: bool + // Directory + dir?: dagger.#Artifact + out: { string @@ -101,6 +104,9 @@ import ( AWS_PAGER: "" } mount: "/cache/aws": "cache" + if dir != _|_ { + mount: "/inputs/source": from: dir + } }, op.#Export & { source: export diff --git a/stdlib/aws/s3/s3.cue b/stdlib/aws/s3/s3.cue index 5c1ef05e..679d2c63 100644 --- a/stdlib/aws/s3/s3.cue +++ b/stdlib/aws/s3/s3.cue @@ -42,20 +42,21 @@ import ( code: #""" opts="" + op=cp if [ -d /inputs/source ]; then - opts="--recursive" + op=sync fi if [ -f /inputs/content_type ]; then opts="--content-type $(cat /inputs/content_type)" fi - aws s3 cp $opts /inputs/source "$(cat /inputs/target)" + aws s3 $op $opts /inputs/source "$(cat /inputs/target)" cat /inputs/target \ | sed -E 's=^s3://([^/]*)/=https://\1.s3.amazonaws.com/=' \ > /url """# if sourceInline == _|_ { - mount: "/inputs/source": from: source + dir: source } } } From b3c29acb3476ffaee8bbbaee599e204d7e40bb8e Mon Sep 17 00:00:00 2001 From: Tom Chauveau Date: Thu, 22 Apr 2021 20:57:00 +0200 Subject: [PATCH 3/3] Add tests on AWS s3 upload directory Signed-off-by: Tom Chauveau --- tests/stdlib.bats | 2 +- tests/stdlib/aws/s3/s3.cue | 23 ++++++++++++++++++++++- tests/stdlib/aws/s3/testdata/bar/foo.txt | 1 + tests/stdlib/aws/s3/testdata/dirFile.txt | 1 + tests/stdlib/aws/s3/verify.cue | 6 ++++-- 5 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 tests/stdlib/aws/s3/testdata/bar/foo.txt create mode 100644 tests/stdlib/aws/s3/testdata/dirFile.txt diff --git a/tests/stdlib.bats b/tests/stdlib.bats index 0d7c2aca..106de816 100644 --- a/tests/stdlib.bats +++ b/tests/stdlib.bats @@ -41,5 +41,5 @@ setup() { @test "stdlib: s3" { skip_unless_secrets_available "$TESTDIR"/stdlib/aws/inputs.yaml - "$DAGGER" compute "$TESTDIR"/stdlib/aws/s3 --input-yaml "$TESTDIR"/stdlib/aws/inputs.yaml + "$DAGGER" compute "$TESTDIR"/stdlib/aws/s3 --input-dir TestDirectory="$TESTDIR"/stdlib/aws/s3/testdata --input-yaml "$TESTDIR"/stdlib/aws/inputs.yaml } \ No newline at end of file diff --git a/tests/stdlib/aws/s3/s3.cue b/tests/stdlib/aws/s3/s3.cue index 92786264..1949c1db 100644 --- a/tests/stdlib/aws/s3/s3.cue +++ b/tests/stdlib/aws/s3/s3.cue @@ -1,6 +1,7 @@ package s3 import ( + "dagger.io/dagger" "dagger.io/aws" "dagger.io/aws/s3" ) @@ -20,5 +21,25 @@ TestS3UploadFile: { target: "s3://\(bucket)/test.txt" } - verify: #VerifyS3 + verify: #VerifyS3 & { + file: "test.txt" + } +} + +TestDirectory: dagger.#Artifact + +TestS3UploadDir: { + deploy: s3.#Put & { + config: TestConfig.awsConfig + source: TestDirectory + target: "s3://\(bucket)/" + } + + verifyFile: #VerifyS3 & { + file: "dirFile.txt" + } + + verifyDir: #VerifyS3 & { + file: "foo.txt" + } } diff --git a/tests/stdlib/aws/s3/testdata/bar/foo.txt b/tests/stdlib/aws/s3/testdata/bar/foo.txt new file mode 100644 index 00000000..616e65a3 --- /dev/null +++ b/tests/stdlib/aws/s3/testdata/bar/foo.txt @@ -0,0 +1 @@ +Test recursivity \ No newline at end of file diff --git a/tests/stdlib/aws/s3/testdata/dirFile.txt b/tests/stdlib/aws/s3/testdata/dirFile.txt new file mode 100644 index 00000000..590f5066 --- /dev/null +++ b/tests/stdlib/aws/s3/testdata/dirFile.txt @@ -0,0 +1 @@ +Test directory \ No newline at end of file diff --git a/tests/stdlib/aws/s3/verify.cue b/tests/stdlib/aws/s3/verify.cue index 90befbcb..b84380d3 100644 --- a/tests/stdlib/aws/s3/verify.cue +++ b/tests/stdlib/aws/s3/verify.cue @@ -19,12 +19,14 @@ import ( // Script aws.#Script & { code: """ - aws s3 ls \(target) > /contents + aws s3 ls --recursive \(target) >> /contents """ } } #VerifyS3: { + file: string + lists: #List & { config: TestConfig.awsConfig target: "s3://\(bucket)" @@ -32,7 +34,7 @@ import ( #CheckFiles: """ - grep -q test.txt /test + grep -q \(file) /test """ test: #up: [