Merge pull request #896 from grouville/localstack

Localstack integration

.github/workflows/ci.yml

@@ -98,6 +98,23 @@ jobs:
     name: Universe
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    services:
+      localstack:
+        image: localstack/localstack:0.12.16
+        env:
+          SERVICES: s3, ecr
+          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+        ports:
+          - 4566:4566
+          - 4571:4571
+          - 4510:4510
+        options: >-
+          --health-cmd "curl -f http://localhost:4566/health"
+          --health-start-period 5s
+          --health-timeout 5s
+          --health-interval 5s
+          --health-retries 10 
+
     steps:
       - name: Check out
         uses: actions/checkout@v2
@@ -117,6 +134,15 @@ jobs:
           mkdir -p ~/.config/dagger
           echo "$DAGGER_AGE_KEY" > ~/.config/dagger/keys.txt
        
+      - name: Provision Localstack AWS resources
+        env:
+            AWS_ACCESS_KEY_ID: test
+            AWS_SECRET_ACCESS_KEY: test
+            AWS_DEFAULT_REGION: us-east-2
+        run: |
+          aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci
+          aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci
+      
       - name: Universe Test
         run: |
           make universe-test

docs/reference/universe/aws/README.md

@@ -21,6 +21,7 @@ Re-usable aws-cli component
 |*config.region*       | `string`             |AWS region             |
 |*config.accessKey*    | `dagger.#Secret`     |AWS access key         |
 |*config.secretKey*    | `dagger.#Secret`     |AWS secret key         |
+|*config.localMode*    | `*null \| string`    |AWS localstack mode    |
 
 ### aws.#CLI Outputs
 
@@ -37,6 +38,7 @@ AWS Config shared by all AWS packages
 |*region*          | `string`             |AWS region             |
 |*accessKey*       | `dagger.#Secret`     |AWS access key         |
 |*secretKey*       | `dagger.#Secret`     |AWS secret key         |
+|*localMode*       | `*null \| string`    |AWS localstack mode    |
 
 ### aws.#Config Outputs
 

docs/reference/universe/aws/cloudformation.md

@@ -21,6 +21,7 @@ AWS CloudFormation Stack
 |*config.region*       | `string`                                     |AWS region                                                       |
 |*config.accessKey*    | `dagger.#Secret`                             |AWS access key                                                   |
 |*config.secretKey*    | `dagger.#Secret`                             |AWS secret key                                                   |
+|*config.localMode*    | `*null \| string`                            |AWS localstack mode                                              |
 |*source*              | `string`                                     |Source is the Cloudformation template (JSON/YAML string)         |
 |*stackName*           | `string`                                     |Stackname is the cloudformation stack                            |
 |*parameters*          | `struct`                                     |Stack parameters                                                 |

docs/reference/universe/aws/ecr.md

@@ -21,9 +21,11 @@ Convert ECR credentials to Docker Login format
 |*config.region*                 | `string`             |AWS region             |
 |*config.accessKey*              | `dagger.#Secret`     |AWS access key         |
 |*config.secretKey*              | `dagger.#Secret`     |AWS secret key         |
+|*config.localMode*              | `*null \| string`    |AWS localstack mode    |
 |*ctr.image.config.region*       | `string`             |AWS region             |
 |*ctr.image.config.accessKey*    | `dagger.#Secret`     |AWS access key         |
 |*ctr.image.config.secretKey*    | `dagger.#Secret`     |AWS secret key         |
+|*ctr.image.config.localMode*    | `*null \| string`    |AWS localstack mode    |
 
 ### ecr.#Credentials Outputs
 

docs/reference/universe/aws/eks.md

@@ -21,6 +21,7 @@ KubeConfig config outputs a valid kube-auth-config for kubectl client
 |*config.region*       | `string`                  |AWS region             |
 |*config.accessKey*    | `dagger.#Secret`          |AWS access key         |
 |*config.secretKey*    | `dagger.#Secret`          |AWS secret key         |
+|*config.localMode*    | `*null \| string`         |AWS localstack mode    |
 |*clusterName*         | `string`                  |EKS cluster name       |
 |*version*             | `*"v1.19.9" \| string`    |Kubectl version        |
 

docs/reference/universe/aws/elb.md

@@ -21,6 +21,7 @@ Returns an unused rule priority (randomized in available range)
 |*config.region*       | `string`             |AWS region             |
 |*config.accessKey*    | `dagger.#Secret`     |AWS access key         |
 |*config.secretKey*    | `dagger.#Secret`     |AWS secret key         |
+|*config.localMode*    | `*null \| string`    |AWS localstack mode    |
 |*listenerArn*         | `string`             |ListenerArn            |
 
 ### elb.#RandomRulePriority Outputs

docs/reference/universe/aws/rds.md

@@ -21,6 +21,7 @@ Creates a new Database on an existing RDS Instance
 |*config.region*       | `string`             |AWS region                                                    |
 |*config.accessKey*    | `dagger.#Secret`     |AWS access key                                                |
 |*config.secretKey*    | `dagger.#Secret`     |AWS secret key                                                |
+|*config.localMode*    | `*null \| string`    |AWS localstack mode                                           |
 |*name*                | `string`             |DB name                                                       |
 |*dbArn*               | `string`             |ARN of the database instance                                  |
 |*secretArn*           | `string`             |ARN of the database secret (for connecting via rds api)       |
@@ -43,6 +44,7 @@ Fetches information on an existing RDS Instance
 |*config.region*       | `string`             |AWS region                      |
 |*config.accessKey*    | `dagger.#Secret`     |AWS access key                  |
 |*config.secretKey*    | `dagger.#Secret`     |AWS secret key                  |
+|*config.localMode*    | `*null \| string`    |AWS localstack mode             |
 |*dbArn*               | `string`             |ARN of the database instance    |
 
 ### rds.#Instance Outputs
@@ -64,6 +66,7 @@ Creates a new user credentials on an existing RDS Instance
 |*config.region*       | `string`             |AWS region                                                    |
 |*config.accessKey*    | `dagger.#Secret`     |AWS access key                                                |
 |*config.secretKey*    | `dagger.#Secret`     |AWS secret key                                                |
+|*config.localMode*    | `*null \| string`    |AWS localstack mode                                           |
 |*username*            | `string`             |Username                                                      |
 |*password*            | `string`             |Password                                                      |
 |*dbArn*               | `string`             |ARN of the database instance                                  |

docs/reference/universe/aws/s3.md

@@ -21,6 +21,7 @@ S3 Bucket object(s) sync
 |*config.region*       | `string`              |AWS region                                                        |
 |*config.accessKey*    | `dagger.#Secret`      |AWS access key                                                    |
 |*config.secretKey*    | `dagger.#Secret`      |AWS secret key                                                    |
+|*config.localMode*    | `*null \| string`     |AWS localstack mode                                               |
 |*source*              | `dagger.#Artifact`    |Source Artifact to upload to S3                                   |
 |*target*              | `string`              |Target S3 URL (eg. s3://\<bucket-name\>/\<path\>/\<sub-path\>)    |
 |*delete*              | `*false \| true`      |Delete files that already exist on remote destination             |

stdlib/.dagger/env/aws-ecr-localstack/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

stdlib/.dagger/env/aws-ecr-localstack/values.yaml

@@ -0,0 +1,30 @@
+plan:
+    package: ./aws/ecr/tests
+name: aws-ecr-localstack
+inputs:
+    TestConfig.awsConfig.accessKey:
+        secret: ENC[AES256_GCM,data:t/iOlA==,iv:oLhubqSKZqn5lVPFn8On//LPj1deAFPXKVTyBKdVODQ=,tag:PORmrxDIf/MHbFDrdxnlTQ==,type:str]
+    TestConfig.awsConfig.localMode:
+        text: "true"
+    TestConfig.awsConfig.secretKey:
+        secret: ENC[AES256_GCM,data:2HYF9w==,iv:f/a9dy4HLJOtKdn8G+zKTboXU1YhzXBEKT1WSvrsCvk=,tag:RKQO+Pg3wg23LIMAnyXhMA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE
+            WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH
+            MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1
+            aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y
+            Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-08-16T20:52:50Z"
+    mac: ENC[AES256_GCM,data:31GBJoqzJOt2q29ql9xTGvhg6XaMupZliAEx2XXD19CvJuAlPdkkbdOP1quEDjHqeBfWUZlTTnqqm2HdSjJ9MgS7aSpgIwb65g/KMez/13QKqlicKaOT55uuyPmWh9+p0rSBCWTHo1vcZldXEuQ9eohwgkV1xHGU3tV+IsmlfrM=,iv:ISdTPiT3rUpZrmMoMBDYmQEB5Q6Tg5fx8leAS3B2kBU=,tag:z/4/2MS6CSlEC++9kCKn5Q==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

stdlib/.dagger/env/aws-s3-localstack/.gitignore

@@ -0,0 +1,2 @@
+# dagger state
+state/**

stdlib/.dagger/env/aws-s3-localstack/values.yaml

@@ -0,0 +1,33 @@
+plan:
+    package: ./aws/s3/tests
+name: aws-s3-localstack
+inputs:
+    TestConfig.awsConfig.accessKey:
+        secret: ENC[AES256_GCM,data:RJY8Cw==,iv:yZW4m74SdRl5qeDfyQVJPOvAktu2qt8jJB45hWUd5p8=,tag:7DCf5ys4Cblb1V1XHMCWAQ==,type:str]
+    TestConfig.awsConfig.localMode:
+        text: "true"
+    TestConfig.awsConfig.secretKey:
+        secret: ENC[AES256_GCM,data:iYxBnA==,iv:7mdUXVnmMJm2pz9Kt1MwrwFeAxwqZaH9o1D2FbUD860=,tag:fj8+JJbNS3Z/L2ft+BtJ1Q==,type:str]
+    TestDirectory:
+        dir:
+            path: ./aws/s3/tests/testdata
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeEk5MS9nVmFoOVNNOHdE
+            WnZCTXBWbW9LL1NJYndCYjhIM2JsNXNEUmxJCkUya0dlZjZ0dGRIM1pVdzg5eWFH
+            MVpiaE9PclNudGdUZm5FcytuVDZGTDAKLS0tIEQxWDdteHgzS3JkdmtNTVpxMUh1
+            aXlvVWJVSGNTSkVyYmpZbi9nUVJZdmMK6csXZ2RMxFw5DB+Hb2TyhyoZT8c2/z7Y
+            Lc9Pe8gb8aUq5Ha+wCybYvY6JWEM5A9XYJKbE7f4borTfGKS72d6pw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-08-16T20:43:09Z"
+    mac: ENC[AES256_GCM,data:suv1ImWFc+7HyrpNw66vasTf75PLu07DivxKCyuqcLFqBBs9qgxg8VFXax1hiZRp7NLTG1cqJ3JLW8DsPzhFTs2xJEDQ8BBKfqNFGgHzucaAISlfKzKKQtPb4NM+56WEmuiF7ns0jKyJH385A1qttD8pVeyLnj4ps+9iU/iA5wo=,iv:LT5XzerWt6ctuPYd/Pm7sdSQni30G5tBNQni+Nfmd3s=,tag:OO+/JhjwcZbnFCIFvOM17g==,type:str]
+    pgp: []
+    encrypted_suffix: secret
+    version: 3.7.1

stdlib/aws/aws.cue

@@ -10,11 +10,13 @@ import (
 // AWS Config shared by all AWS packages
 #Config: {
 	// AWS region
-	region: string @dagger(input)
+	region: dagger.#Input & {string}
 	// AWS access key
-	accessKey: dagger.#Secret @dagger(input)
+	accessKey: dagger.#Input & {dagger.#Secret}
 	// AWS secret key
-	secretKey: dagger.#Secret @dagger(input)
+	secretKey: dagger.#Input & {dagger.#Secret}
+	// AWS localstack mode
+	localMode: dagger.#Input & {string | *null}
 }
 
 // Re-usable aws-cli component
@@ -30,6 +32,9 @@ import (
 				"package": jq:        "=~1.6"
 				"package": curl:      true
 				"package": "aws-cli": "=~1.18"
+				if config.localMode != null {
+					package: "py3-pip": true
+				}
 			}
 		},
 		op.#Exec & {
@@ -40,6 +45,7 @@ import (
 				"-eo",
 				"pipefail",
 				"-c",
+				if config.localMode == null {
 					#"""
 						aws configure set aws_access_key_id "$(cat /run/secrets/access_key)"
 						aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)"
@@ -47,7 +53,29 @@ import (
 						aws configure set default.region "$AWS_DEFAULT_REGION"
 						aws configure set default.cli_pager ""
 						aws configure set default.output "json"
-					"""#,
+						"""#
+				},
+				if config.localMode != null {
+					#"""
+						# Download awscli v3 and override aws
+						pip install awscli-local[v2]
+						mv /usr/bin/awslocal /usr/bin/aws
+
+						# Configure
+						mkdir -p ~/.aws/
+
+						# Set up ~/.aws/config
+						echo "[default]" > ~/.aws/config
+						echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config
+						echo "cli_pager =" >> ~/.aws/config
+						echo "output = json" >> ~/.aws/config
+
+						# Set up ~/.aws/credentials
+						echo "[default]" > ~/.aws/credentials
+						echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials
+						echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials
+						"""#
+				},
 			]
 			mount: "/run/secrets/access_key": secret: config.accessKey
 			mount: "/run/secrets/secret_key": secret: config.secretKey

stdlib/aws/ecr/tests/ecr.cue

@@ -11,11 +11,19 @@ TestConfig: awsConfig: aws.#Config & {
 }
 
 TestECR: {
+	localMode: TestConfig.awsConfig.localMode
+
 	suffix: random.#String & {
 		seed: ""
 	}
 
+	repository: string
+	if localMode == null {
 		repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci"
+	}
+	if localMode != null {
+		repository: "localhost:4510/dagger-ci"
+	}
 	tag: "test-ecr-\(suffix.out)"
 
 	creds: #Credentials & {

stdlib/helpers.bash

@@ -1,3 +1,20 @@
+# Instead of setup, this just runs once
+setup_file() {
+    # Cleanup local Localstack instances
+    if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ] && \
+        [ "$GITHUB_ACTIONS" != "true" ]; then
+        echo "Cleanup local LOCALSTACK"
+        
+        # S3 buckets cleanup
+        aws --endpoint-url=http://localhost:4566 s3 rm s3://dagger-ci 2>/dev/null || true
+        aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci 2>/dev/null || true
+        
+        # ECR repositories cleanup
+        aws --endpoint-url=http://localhost:4566 ecr delete-repository --repository-name dagger-ci 2>/dev/null || true
+        aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci 2>/dev/null || true
+    fi
+}
+
 common_setup() {
     load 'node_modules/bats-support/load'
     load 'node_modules/bats-assert/load'
@@ -58,6 +75,16 @@ copy_to_sandbox() {
       cp -a "$source_package" "$target_package"
     fi
 }
+# Check if there is a localstack instance.
+#
+# This is needed to do docs test in the CI.
+skip_unless_local_localstack() {
+    if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then
+        echo "Localstack available"
+    else
+        skip "Localstack not available"
+    fi
+}
 
 # Check if there is a local kubernetes cluster.
 #

stdlib/universe.bats

@@ -45,10 +45,22 @@ setup() {
     dagger -e aws-ecr up
 }
 
+@test "aws: ecr/localstack" {
+    skip_unless_local_localstack
+
+    dagger -e aws-ecr-localstack up
+}
+
 @test "aws: s3" {
     dagger -e aws-s3 up
 }
 
+@test "aws: s3/localstack" {
+    skip_unless_local_localstack
+
+    dagger -e aws-s3-localstack up
+}
+
 @test "aws: eks" {
     dagger -e aws-eks up
 }