diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75a91212..a31984b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,23 @@ jobs: name: Universe runs-on: ubuntu-latest timeout-minutes: 30 + services: + localstack: + image: localstack/localstack:0.12.16 + env: + SERVICES: s3, ecr + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + ports: + - 4566:4566 + - 4571:4571 + - 4510:4510 + options: >- + --health-cmd "curl -f http://localhost:4566/health" + --health-start-period 5s + --health-timeout 5s + --health-interval 5s + --health-retries 10 + steps: - name: Check out uses: actions/checkout@v2 @@ -116,7 +133,16 @@ jobs: run: | mkdir -p ~/.config/dagger echo "$DAGGER_AGE_KEY" > ~/.config/dagger/keys.txt - + + - name: Provision Localstack AWS resources + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-2 + run: | + aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci + aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci + - name: Universe Test run: | make universe-test diff --git a/docs/reference/universe/aws/README.md b/docs/reference/universe/aws/README.md index 7484c97e..ad23b78d 100644 --- a/docs/reference/universe/aws/README.md +++ b/docs/reference/universe/aws/README.md @@ -16,11 +16,12 @@ Re-usable aws-cli component ### aws.#CLI Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | ### aws.#CLI Outputs @@ -32,11 +33,12 @@ AWS Config shared by all AWS packages ### aws.#Config Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*region* | `string` |AWS region | -|*accessKey* | `dagger.#Secret` |AWS access key | -|*secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*region* | `string` |AWS region | +|*accessKey* | `dagger.#Secret` |AWS access key | +|*secretKey* | `dagger.#Secret` |AWS secret key | +|*localMode* | `*null \| string` |AWS localstack mode | ### aws.#Config Outputs diff --git a/docs/reference/universe/aws/cloudformation.md b/docs/reference/universe/aws/cloudformation.md index 8528db70..11b81e17 100644 --- a/docs/reference/universe/aws/cloudformation.md +++ b/docs/reference/universe/aws/cloudformation.md @@ -21,6 +21,7 @@ AWS CloudFormation Stack |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `string` |Source is the Cloudformation template (JSON/YAML string) | |*stackName* | `string` |Stackname is the cloudformation stack | |*parameters* | `struct` |Stack parameters | diff --git a/docs/reference/universe/aws/ecr.md b/docs/reference/universe/aws/ecr.md index c45d1824..bbfcd619 100644 --- a/docs/reference/universe/aws/ecr.md +++ b/docs/reference/universe/aws/ecr.md @@ -16,14 +16,16 @@ Convert ECR credentials to Docker Login format ### ecr.#Credentials Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*ctr.image.config.region* | `string` |AWS region | -|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | -|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*ctr.image.config.region* | `string` |AWS region | +|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | +|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +|*ctr.image.config.localMode* | `*null \| string` |AWS localstack mode | ### ecr.#Credentials Outputs diff --git a/docs/reference/universe/aws/eks.md b/docs/reference/universe/aws/eks.md index 7d445769..f7ca7423 100644 --- a/docs/reference/universe/aws/eks.md +++ b/docs/reference/universe/aws/eks.md @@ -16,13 +16,14 @@ KubeConfig config outputs a valid kube-auth-config for kubectl client ### eks.#KubeConfig Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*clusterName* | `string` |EKS cluster name | -|*version* | `*"v1.19.9" \| string` |Kubectl version | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*clusterName* | `string` |EKS cluster name | +|*version* | `*"v1.19.9" \| string` |Kubectl version | ### eks.#KubeConfig Outputs diff --git a/docs/reference/universe/aws/elb.md b/docs/reference/universe/aws/elb.md index adf122a4..97142576 100644 --- a/docs/reference/universe/aws/elb.md +++ b/docs/reference/universe/aws/elb.md @@ -16,12 +16,13 @@ Returns an unused rule priority (randomized in available range) ### elb.#RandomRulePriority Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*listenerArn* | `string` |ListenerArn | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*listenerArn* | `string` |ListenerArn | ### elb.#RandomRulePriority Outputs diff --git a/docs/reference/universe/aws/rds.md b/docs/reference/universe/aws/rds.md index 9c5735f1..df58a790 100644 --- a/docs/reference/universe/aws/rds.md +++ b/docs/reference/universe/aws/rds.md @@ -16,15 +16,16 @@ Creates a new Database on an existing RDS Instance ### rds.#Database Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*name* | `string` |DB name | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*name* | `string` |DB name | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#Database Outputs @@ -38,12 +39,13 @@ Fetches information on an existing RDS Instance ### rds.#Instance Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*dbArn* | `string` |ARN of the database instance | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*dbArn* | `string` |ARN of the database instance | ### rds.#Instance Outputs @@ -59,17 +61,18 @@ Creates a new user credentials on an existing RDS Instance ### rds.#User Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*username* | `string` |Username | -|*password* | `string` |Password | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*username* | `string` |Username | +|*password* | `string` |Password | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#User Outputs diff --git a/docs/reference/universe/aws/s3.md b/docs/reference/universe/aws/s3.md index ca7b54dd..0b1c2431 100644 --- a/docs/reference/universe/aws/s3.md +++ b/docs/reference/universe/aws/s3.md @@ -21,6 +21,7 @@ S3 Bucket object(s) sync |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `dagger.#Artifact` |Source Artifact to upload to S3 | |*target* | `string` |Target S3 URL (eg. s3://\/\/\) | |*delete* | `*false \| true` |Delete files that already exist on remote destination | diff --git a/stdlib/.dagger/env/aws-ecr-localstack/.gitignore b/stdlib/.dagger/env/aws-ecr-localstack/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/aws-ecr-localstack/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/aws-ecr-localstack/values.yaml b/stdlib/.dagger/env/aws-ecr-localstack/values.yaml new file mode 100644 index 00000000..4dc687ba --- /dev/null +++ b/stdlib/.dagger/env/aws-ecr-localstack/values.yaml @@ -0,0 +1,30 @@ +plan: + package: ./aws/ecr/tests +name: aws-ecr-localstack +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:t/iOlA==,iv:oLhubqSKZqn5lVPFn8On//LPj1deAFPXKVTyBKdVODQ=,tag:PORmrxDIf/MHbFDrdxnlTQ==,type:str] + TestConfig.awsConfig.localMode: + text: "true" + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:2HYF9w==,iv:f/a9dy4HLJOtKdn8G+zKTboXU1YhzXBEKT1WSvrsCvk=,tag:RKQO+Pg3wg23LIMAnyXhMA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-08-16T20:52:50Z" + mac: ENC[AES256_GCM,data:31GBJoqzJOt2q29ql9xTGvhg6XaMupZliAEx2XXD19CvJuAlPdkkbdOP1quEDjHqeBfWUZlTTnqqm2HdSjJ9MgS7aSpgIwb65g/KMez/13QKqlicKaOT55uuyPmWh9+p0rSBCWTHo1vcZldXEuQ9eohwgkV1xHGU3tV+IsmlfrM=,iv:ISdTPiT3rUpZrmMoMBDYmQEB5Q6Tg5fx8leAS3B2kBU=,tag:z/4/2MS6CSlEC++9kCKn5Q==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/.dagger/env/aws-s3-localstack/.gitignore b/stdlib/.dagger/env/aws-s3-localstack/.gitignore new file mode 100644 index 00000000..01ec19b0 --- /dev/null +++ b/stdlib/.dagger/env/aws-s3-localstack/.gitignore @@ -0,0 +1,2 @@ +# dagger state +state/** diff --git a/stdlib/.dagger/env/aws-s3-localstack/values.yaml b/stdlib/.dagger/env/aws-s3-localstack/values.yaml new file mode 100644 index 00000000..22c4e683 --- /dev/null +++ b/stdlib/.dagger/env/aws-s3-localstack/values.yaml @@ -0,0 +1,33 @@ +plan: + package: ./aws/s3/tests +name: aws-s3-localstack +inputs: + TestConfig.awsConfig.accessKey: + secret: ENC[AES256_GCM,data:RJY8Cw==,iv:yZW4m74SdRl5qeDfyQVJPOvAktu2qt8jJB45hWUd5p8=,tag:7DCf5ys4Cblb1V1XHMCWAQ==,type:str] + TestConfig.awsConfig.localMode: + text: "true" + TestConfig.awsConfig.secretKey: + secret: ENC[AES256_GCM,data:iYxBnA==,iv:7mdUXVnmMJm2pz9Kt1MwrwFeAxwqZaH9o1D2FbUD860=,tag:fj8+JJbNS3Z/L2ft+BtJ1Q==,type:str] + TestDirectory: + dir: + path: ./aws/s3/tests/testdata +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE + WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH + MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1 + aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y + Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2021-08-16T20:43:09Z" + mac: ENC[AES256_GCM,data:suv1ImWFc+7HyrpNw66vasTf75PLu07DivxKCyuqcLFqBBs9qgxg8VFXax1hiZRp7NLTG1cqJ3JLW8DsPzhFTs2xJEDQ8BBKfqNFGgHzucaAISlfKzKKQtPb4NM+56WEmuiF7ns0jKyJH385A1qttD8pVeyLnj4ps+9iU/iA5wo=,iv:LT5XzerWt6ctuPYd/Pm7sdSQni30G5tBNQni+Nfmd3s=,tag:OO+/JhjwcZbnFCIFvOM17g==,type:str] + pgp: [] + encrypted_suffix: secret + version: 3.7.1 diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index 2e0a3ba5..c3e24fec 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -10,11 +10,13 @@ import ( // AWS Config shared by all AWS packages #Config: { // AWS region - region: string @dagger(input) + region: dagger.#Input & {string} // AWS access key - accessKey: dagger.#Secret @dagger(input) + accessKey: dagger.#Input & {dagger.#Secret} // AWS secret key - secretKey: dagger.#Secret @dagger(input) + secretKey: dagger.#Input & {dagger.#Secret} + // AWS localstack mode + localMode: dagger.#Input & {string | *null} } // Re-usable aws-cli component @@ -30,6 +32,9 @@ import ( "package": jq: "=~1.6" "package": curl: true "package": "aws-cli": "=~1.18" + if config.localMode != null { + package: "py3-pip": true + } } }, op.#Exec & { @@ -40,14 +45,37 @@ import ( "-eo", "pipefail", "-c", - #""" - aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" - aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" + if config.localMode == null { + #""" + aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" + aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" - aws configure set default.region "$AWS_DEFAULT_REGION" - aws configure set default.cli_pager "" - aws configure set default.output "json" - """#, + aws configure set default.region "$AWS_DEFAULT_REGION" + aws configure set default.cli_pager "" + aws configure set default.output "json" + """# + }, + if config.localMode != null { + #""" + # Download awscli v3 and override aws + pip install awscli-local[v2] + mv /usr/bin/awslocal /usr/bin/aws + + # Configure + mkdir -p ~/.aws/ + + # Set up ~/.aws/config + echo "[default]" > ~/.aws/config + echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config + echo "cli_pager =" >> ~/.aws/config + echo "output = json" >> ~/.aws/config + + # Set up ~/.aws/credentials + echo "[default]" > ~/.aws/credentials + echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials + echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials + """# + }, ] mount: "/run/secrets/access_key": secret: config.accessKey mount: "/run/secrets/secret_key": secret: config.secretKey diff --git a/stdlib/aws/ecr/tests/ecr.cue b/stdlib/aws/ecr/tests/ecr.cue index c78a68eb..d9b99c45 100644 --- a/stdlib/aws/ecr/tests/ecr.cue +++ b/stdlib/aws/ecr/tests/ecr.cue @@ -11,12 +11,20 @@ TestConfig: awsConfig: aws.#Config & { } TestECR: { + localMode: TestConfig.awsConfig.localMode + suffix: random.#String & { seed: "" } - repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" - tag: "test-ecr-\(suffix.out)" + repository: string + if localMode == null { + repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" + } + if localMode != null { + repository: "localhost:4510/dagger-ci" + } + tag: "test-ecr-\(suffix.out)" creds: #Credentials & { config: TestConfig.awsConfig diff --git a/stdlib/helpers.bash b/stdlib/helpers.bash index f2f4b4c7..5df6078f 100644 --- a/stdlib/helpers.bash +++ b/stdlib/helpers.bash @@ -1,3 +1,20 @@ +# Instead of setup, this just runs once +setup_file() { + # Cleanup local Localstack instances + if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ] && \ + [ "$GITHUB_ACTIONS" != "true" ]; then + echo "Cleanup local LOCALSTACK" + + # S3 buckets cleanup + aws --endpoint-url=http://localhost:4566 s3 rm s3://dagger-ci 2>/dev/null || true + aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci 2>/dev/null || true + + # ECR repositories cleanup + aws --endpoint-url=http://localhost:4566 ecr delete-repository --repository-name dagger-ci 2>/dev/null || true + aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci 2>/dev/null || true + fi +} + common_setup() { load 'node_modules/bats-support/load' load 'node_modules/bats-assert/load' @@ -58,6 +75,16 @@ copy_to_sandbox() { cp -a "$source_package" "$target_package" fi } +# Check if there is a localstack instance. +# +# This is needed to do docs test in the CI. +skip_unless_local_localstack() { + if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then + echo "Localstack available" + else + skip "Localstack not available" + fi +} # Check if there is a local kubernetes cluster. # diff --git a/stdlib/universe.bats b/stdlib/universe.bats index bfb92323..ddc2379b 100644 --- a/stdlib/universe.bats +++ b/stdlib/universe.bats @@ -45,10 +45,22 @@ setup() { dagger -e aws-ecr up } +@test "aws: ecr/localstack" { + skip_unless_local_localstack + + dagger -e aws-ecr-localstack up +} + @test "aws: s3" { dagger -e aws-s3 up } +@test "aws: s3/localstack" { + skip_unless_local_localstack + + dagger -e aws-s3-localstack up +} + @test "aws: eks" { dagger -e aws-eks up }