From 58209b87d108f43342f7d46c36ba50d6d5c6917a Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Thu, 19 Aug 2021 17:11:33 +0200 Subject: [PATCH] Doc update + linting + fix CI typo + fix port issue ECR Signed-off-by: Guillaume de Rouville --- .github/workflows/ci.yml | 5 +- docs/reference/universe/aws/README.md | 22 ++++--- docs/reference/universe/aws/cloudformation.md | 1 + docs/reference/universe/aws/ecr.md | 18 +++--- docs/reference/universe/aws/eks.md | 15 ++--- docs/reference/universe/aws/elb.md | 13 ++-- docs/reference/universe/aws/rds.md | 55 ++++++++-------- docs/reference/universe/aws/s3.md | 1 + stdlib/aws/aws.cue | 62 +++++++++---------- stdlib/aws/ecr/tests/ecr.cue | 6 +- stdlib/helpers.bash | 4 +- 11 files changed, 107 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 782cd50a..a31984b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,13 +100,14 @@ jobs: timeout-minutes: 30 services: localstack: - image: localstack/localstack:latest + image: localstack/localstack:0.12.16 env: SERVICES: s3, ecr LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} ports: - 4566:4566 - 4571:4571 + - 4510:4510 options: >- --health-cmd "curl -f http://localhost:4566/health" --health-start-period 5s @@ -137,7 +138,7 @@ jobs: env: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test - AWS_DEFAULT_REGION: eu-east-2 + AWS_DEFAULT_REGION: us-east-2 run: | aws --endpoint-url=http://localhost:4566 s3 mb s3://dagger-ci aws --endpoint-url=http://localhost:4566 ecr create-repository --repository-name dagger-ci diff --git a/docs/reference/universe/aws/README.md b/docs/reference/universe/aws/README.md index 7484c97e..ad23b78d 100644 --- a/docs/reference/universe/aws/README.md +++ b/docs/reference/universe/aws/README.md @@ -16,11 +16,12 @@ Re-usable aws-cli component ### aws.#CLI Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | ### aws.#CLI Outputs @@ -32,11 +33,12 @@ AWS Config shared by all AWS packages ### aws.#Config Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*region* | `string` |AWS region | -|*accessKey* | `dagger.#Secret` |AWS access key | -|*secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*region* | `string` |AWS region | +|*accessKey* | `dagger.#Secret` |AWS access key | +|*secretKey* | `dagger.#Secret` |AWS secret key | +|*localMode* | `*null \| string` |AWS localstack mode | ### aws.#Config Outputs diff --git a/docs/reference/universe/aws/cloudformation.md b/docs/reference/universe/aws/cloudformation.md index 8528db70..11b81e17 100644 --- a/docs/reference/universe/aws/cloudformation.md +++ b/docs/reference/universe/aws/cloudformation.md @@ -21,6 +21,7 @@ AWS CloudFormation Stack |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `string` |Source is the Cloudformation template (JSON/YAML string) | |*stackName* | `string` |Stackname is the cloudformation stack | |*parameters* | `struct` |Stack parameters | diff --git a/docs/reference/universe/aws/ecr.md b/docs/reference/universe/aws/ecr.md index c45d1824..bbfcd619 100644 --- a/docs/reference/universe/aws/ecr.md +++ b/docs/reference/universe/aws/ecr.md @@ -16,14 +16,16 @@ Convert ECR credentials to Docker Login format ### ecr.#Credentials Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*ctr.image.config.region* | `string` |AWS region | -|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | -|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*ctr.image.config.region* | `string` |AWS region | +|*ctr.image.config.accessKey* | `dagger.#Secret` |AWS access key | +|*ctr.image.config.secretKey* | `dagger.#Secret` |AWS secret key | +|*ctr.image.config.localMode* | `*null \| string` |AWS localstack mode | ### ecr.#Credentials Outputs diff --git a/docs/reference/universe/aws/eks.md b/docs/reference/universe/aws/eks.md index 7d445769..f7ca7423 100644 --- a/docs/reference/universe/aws/eks.md +++ b/docs/reference/universe/aws/eks.md @@ -16,13 +16,14 @@ KubeConfig config outputs a valid kube-auth-config for kubectl client ### eks.#KubeConfig Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*clusterName* | `string` |EKS cluster name | -|*version* | `*"v1.19.9" \| string` |Kubectl version | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*clusterName* | `string` |EKS cluster name | +|*version* | `*"v1.19.9" \| string` |Kubectl version | ### eks.#KubeConfig Outputs diff --git a/docs/reference/universe/aws/elb.md b/docs/reference/universe/aws/elb.md index adf122a4..97142576 100644 --- a/docs/reference/universe/aws/elb.md +++ b/docs/reference/universe/aws/elb.md @@ -16,12 +16,13 @@ Returns an unused rule priority (randomized in available range) ### elb.#RandomRulePriority Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*listenerArn* | `string` |ListenerArn | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*listenerArn* | `string` |ListenerArn | ### elb.#RandomRulePriority Outputs diff --git a/docs/reference/universe/aws/rds.md b/docs/reference/universe/aws/rds.md index 9c5735f1..df58a790 100644 --- a/docs/reference/universe/aws/rds.md +++ b/docs/reference/universe/aws/rds.md @@ -16,15 +16,16 @@ Creates a new Database on an existing RDS Instance ### rds.#Database Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*name* | `string` |DB name | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*name* | `string` |DB name | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#Database Outputs @@ -38,12 +39,13 @@ Fetches information on an existing RDS Instance ### rds.#Instance Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*dbArn* | `string` |ARN of the database instance | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*dbArn* | `string` |ARN of the database instance | ### rds.#Instance Outputs @@ -59,17 +61,18 @@ Creates a new user credentials on an existing RDS Instance ### rds.#User Inputs -| Name | Type | Description | -| ------------- |:-------------: |:-------------: | -|*config.region* | `string` |AWS region | -|*config.accessKey* | `dagger.#Secret` |AWS access key | -|*config.secretKey* | `dagger.#Secret` |AWS secret key | -|*username* | `string` |Username | -|*password* | `string` |Password | -|*dbArn* | `string` |ARN of the database instance | -|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | -|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | -|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | +| Name | Type | Description | +| ------------- |:-------------: |:-------------: | +|*config.region* | `string` |AWS region | +|*config.accessKey* | `dagger.#Secret` |AWS access key | +|*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | +|*username* | `string` |Username | +|*password* | `string` |Password | +|*dbArn* | `string` |ARN of the database instance | +|*secretArn* | `string` |ARN of the database secret (for connecting via rds api) | +|*grantDatabase* | `*"" \| string` |Name of the database to grants access to | +|*dbType* | `string` |Database type MySQL or PostgreSQL (Aurora Serverless only) | ### rds.#User Outputs diff --git a/docs/reference/universe/aws/s3.md b/docs/reference/universe/aws/s3.md index ca7b54dd..0b1c2431 100644 --- a/docs/reference/universe/aws/s3.md +++ b/docs/reference/universe/aws/s3.md @@ -21,6 +21,7 @@ S3 Bucket object(s) sync |*config.region* | `string` |AWS region | |*config.accessKey* | `dagger.#Secret` |AWS access key | |*config.secretKey* | `dagger.#Secret` |AWS secret key | +|*config.localMode* | `*null \| string` |AWS localstack mode | |*source* | `dagger.#Artifact` |Source Artifact to upload to S3 | |*target* | `string` |Target S3 URL (eg. s3://\/\/\) | |*delete* | `*false \| true` |Delete files that already exist on remote destination | diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue index 8c462c7a..c3e24fec 100644 --- a/stdlib/aws/aws.cue +++ b/stdlib/aws/aws.cue @@ -10,13 +10,13 @@ import ( // AWS Config shared by all AWS packages #Config: { // AWS region - region: dagger.#Input & { string } + region: dagger.#Input & {string} // AWS access key - accessKey: dagger.#Input & { dagger.#Secret } + accessKey: dagger.#Input & {dagger.#Secret} // AWS secret key - secretKey: dagger.#Input & { dagger.#Secret } + secretKey: dagger.#Input & {dagger.#Secret} // AWS localstack mode - localMode: dagger.#Input & { string | *null } + localMode: dagger.#Input & {string | *null} } // Re-usable aws-cli component @@ -33,7 +33,7 @@ import ( "package": curl: true "package": "aws-cli": "=~1.18" if config.localMode != null { - "package": "py3-pip": true + package: "py3-pip": true } } }, @@ -46,36 +46,36 @@ import ( "pipefail", "-c", if config.localMode == null { - #""" - aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" - aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" + #""" + aws configure set aws_access_key_id "$(cat /run/secrets/access_key)" + aws configure set aws_secret_access_key "$(cat /run/secrets/secret_key)" - aws configure set default.region "$AWS_DEFAULT_REGION" - aws configure set default.cli_pager "" - aws configure set default.output "json" - """#, - } - if config.localMode != null { - #""" - # Download awscli v3 and override aws - pip install awscli-local[v2] - mv /usr/bin/awslocal /usr/bin/aws + aws configure set default.region "$AWS_DEFAULT_REGION" + aws configure set default.cli_pager "" + aws configure set default.output "json" + """# + }, + if config.localMode != null { + #""" + # Download awscli v3 and override aws + pip install awscli-local[v2] + mv /usr/bin/awslocal /usr/bin/aws - # Configure - mkdir -p ~/.aws/ + # Configure + mkdir -p ~/.aws/ - # Set up ~/.aws/config - echo "[default]" > ~/.aws/config - echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config - echo "cli_pager =" >> ~/.aws/config - echo "output = json" >> ~/.aws/config + # Set up ~/.aws/config + echo "[default]" > ~/.aws/config + echo "region = $AWS_DEFAULT_REGION" >> ~/.aws/config + echo "cli_pager =" >> ~/.aws/config + echo "output = json" >> ~/.aws/config - # Set up ~/.aws/credentials - echo "[default]" > ~/.aws/credentials - echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials - echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials - """#, - } + # Set up ~/.aws/credentials + echo "[default]" > ~/.aws/credentials + echo "aws_access_key_id = $(cat /run/secrets/access_key)" >> ~/.aws/credentials + echo "aws_secret_access_key = $(cat /run/secrets/secret_key)" >> ~/.aws/credentials + """# + }, ] mount: "/run/secrets/access_key": secret: config.accessKey mount: "/run/secrets/secret_key": secret: config.secretKey diff --git a/stdlib/aws/ecr/tests/ecr.cue b/stdlib/aws/ecr/tests/ecr.cue index 561f2cd7..d9b99c45 100644 --- a/stdlib/aws/ecr/tests/ecr.cue +++ b/stdlib/aws/ecr/tests/ecr.cue @@ -17,14 +17,14 @@ TestECR: { seed: "" } - repository: string + repository: string if localMode == null { repository: "125635003186.dkr.ecr.\(TestConfig.awsConfig.region).amazonaws.com/dagger-ci" } if localMode != null { - repository: "localhost:4514/dagger-ci" + repository: "localhost:4510/dagger-ci" } - tag: "test-ecr-\(suffix.out)" + tag: "test-ecr-\(suffix.out)" creds: #Credentials & { config: TestConfig.awsConfig diff --git a/stdlib/helpers.bash b/stdlib/helpers.bash index 94707528..436911fe 100644 --- a/stdlib/helpers.bash +++ b/stdlib/helpers.bash @@ -58,9 +58,9 @@ copy_to_sandbox() { cp -a "$source_package" "$target_package" fi } -# Check if there is a local kubernetes cluster. +# Check if there is a localstack instance. # -# This is need to do kubernetes test in the CI. +# This is needed to do docs test in the CI. skip_unless_local_localstack() { if [ "$(curl -s http://localhost:4566)" = '{"status": "running"}' ]; then echo "Localstack available"