From ee7bb3c3f5ff1599fd040f36243dc57cc64fa00d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 3 Mar 2021 17:24:21 -0800 Subject: [PATCH 1/5] implemented "write-file" and "mkdir" ops Signed-off-by: Sam Alba --- dagger/compiler/value.go | 5 ++++ dagger/pipeline.go | 57 ++++++++++++++++++++++++++++++++++++++++ stdlib/dagger/dagger.cue | 14 ++++++++++ 3 files changed, 76 insertions(+) diff --git a/dagger/compiler/value.go b/dagger/compiler/value.go index f5c8ee85..e04db9a3 100644 --- a/dagger/compiler/value.go +++ b/dagger/compiler/value.go @@ -92,6 +92,11 @@ func (v *Value) String() (string, error) { return v.val.String() } +// Proxy function to the underlying cue.Value +func (v *Value) Int64() (int64, error) { + return v.val.Int64() +} + func (v *Value) SourceUnsafe() string { s, _ := v.SourceString() return s diff --git a/dagger/pipeline.go b/dagger/pipeline.go index 6b8784ab..10c5bdd0 100644 --- a/dagger/pipeline.go +++ b/dagger/pipeline.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "strings" "github.com/docker/distribution/reference" @@ -167,6 +168,10 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error { return p.Subdir(ctx, op) case "docker-build": return p.DockerBuild(ctx, op) + case "write-file": + return p.WriteFile(ctx, op) + case "mkdir": + return p.Mkdir(ctx, op) default: return fmt.Errorf("invalid operation: %s", op.JSON()) } @@ -679,3 +684,55 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error { return nil } + +func (p *Pipeline) WriteFile(ctx context.Context, op *compiler.Value) error { + content, err := op.Get("content").String() + if err != nil { + return err + } + + dest, err := op.Get("dest").String() + if err != nil { + return err + } + + mode, err := op.Get("mode").Int64() + if err != nil { + return err + } + + p.fs = p.fs.Change(func(st llb.State) llb.State { + return st.File( + llb.Mkfile(dest, fs.FileMode(mode), []byte(content)), + llb.WithCustomName(p.vertexNamef("WriteFile %s", dest)), + ) + }) + + return nil +} + +func (p *Pipeline) Mkdir(ctx context.Context, op *compiler.Value) error { + path, err := op.Get("path").String() + if err != nil { + return err + } + + dir, err := op.Get("dir").String() + if err != nil { + return err + } + + mode, err := op.Get("mode").Int64() + if err != nil { + return err + } + + p.fs = p.fs.Change(func(st llb.State) llb.State { + return st.Dir(dir).File( + llb.Mkdir(path, fs.FileMode(mode)), + llb.WithCustomName(p.vertexNamef("Mkdir %s", path)), + ) + }) + + return nil +} diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index ecdc79c5..3fbcc721 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -71,3 +71,17 @@ package dagger buildArg?: [string]: string label?: [string]: string } + +#WriteFile: { + do: "write-file" + content: string + dest: string + mode: int | *0o644 +} + +#Mkdir: { + do: "mkdir" + dir: *"/" | string + path: string + mode: int | *0o755 +} From 98a06c67d21c34e70cbba75e80abd8cbeaa94dc2 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 3 Mar 2021 17:24:45 -0800 Subject: [PATCH 2/5] add support for sub-dirs packages in stdlib Signed-off-by: Sam Alba --- stdlib/stdlib.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 4f10f222..8ec97e80 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -11,7 +11,7 @@ import ( ) // FS contains the filesystem of the stdlib. -//go:embed **/*.cue +//go:embed **/*.cue **/*/*.cue var FS embed.FS const ( From 64ab495c82c66c20ae69915294df77823536ff8f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 3 Mar 2021 17:33:19 -0800 Subject: [PATCH 3/5] implemented aws.#Config and aws/cloudformation packages in stdlib Signed-off-by: Sam Alba --- stdlib/aws/aws.cue | 10 ++ stdlib/aws/cloudformation/cloudformation.cue | 101 +++++++++++++++++ stdlib/aws/cloudformation/code.cue | 108 +++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 stdlib/aws/aws.cue create mode 100644 stdlib/aws/cloudformation/cloudformation.cue create mode 100644 stdlib/aws/cloudformation/code.cue diff --git a/stdlib/aws/aws.cue b/stdlib/aws/aws.cue new file mode 100644 index 00000000..479acfb5 --- /dev/null +++ b/stdlib/aws/aws.cue @@ -0,0 +1,10 @@ +package aws + +#Config: { + // AWS region + region: string + // AWS access key + accessKey: string // FIXME: should be a secret + // AWS secret key + secretKey: string // FIXME: should be a secret +} diff --git a/stdlib/aws/cloudformation/cloudformation.cue b/stdlib/aws/cloudformation/cloudformation.cue new file mode 100644 index 00000000..aa6c2f79 --- /dev/null +++ b/stdlib/aws/cloudformation/cloudformation.cue @@ -0,0 +1,101 @@ +package cloudformation + +import ( + "encoding/json" + + "dagger.io/dagger" + "dagger.io/alpine" + "dagger.io/aws" +) + +// AWS CloudFormation Stack +#Stack: { + + // AWS Config + config: aws.#Config + + // Source is the Cloudformation template (JSON/YAML string) + source: string + + // Stackname is the cloudformation stack + stackName: string + + // Stack parameters + parameters: [string]: _ + + // Behavior when failure to create/update the Stack + onFailure: *"DO_NOTHING" | "ROLLBACK" | "DELETE" + + // Timeout for waiting for the stack to be created/updated (in minutes) + timeout: *10 | uint + + // Never update the stack if already exists + neverUpdate: *false | bool + + #files: { + "/entrypoint.sh": #Code + "/src/template.json": source + if len(parameters) > 0 { + "/src/parameters.json": json.Marshal( + [ for key, val in parameters { + ParameterKey: "\(key)" + ParameterValue: "\(val)" + }]) + "/src/parameters_overrides.json": json.Marshal([ for key, val in parameters {"\(key)=\(val)"}]) + } + } + + outputs: { + [string]: string + + #dagger: compute: [ + dagger.#Load & { + from: alpine.#Image & { + package: bash: "=5.1.0-r0" + package: jq: "=1.6-r1" + package: "aws-cli": "=1.18.177-r0" + } + }, + dagger.#Mkdir & { + path: "/src" + }, + for dest, content in #files { + dagger.#WriteFile & { + "dest": dest + "content": content + } + }, + dagger.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "/entrypoint.sh", + ] + env: { + AWS_CONFIG_FILE: "/cache/aws/config" + AWS_ACCESS_KEY_ID: config.accessKey + AWS_SECRET_ACCESS_KEY: config.secretKey + AWS_DEFAULT_REGION: config.region + AWS_REGION: config.region + AWS_DEFAULT_OUTPUT: "json" + AWS_PAGER: "" + if neverUpdate { + NEVER_UPDATE: "true" + } + STACK_NAME: stackName + TIMEOUT: "\(timeout)" + ON_FAILURE: onFailure + } + dir: "/src" + mount: "/cache/aws": "cache" + }, + dagger.#Export & { + source: "/outputs.json" + format: "json" + }, + ] + } +} diff --git a/stdlib/aws/cloudformation/code.cue b/stdlib/aws/cloudformation/code.cue new file mode 100644 index 00000000..7d0d3c6b --- /dev/null +++ b/stdlib/aws/cloudformation/code.cue @@ -0,0 +1,108 @@ +package cloudformation + +#Code: #""" + set +o pipefail + + aws cloudformation validate-template --template-body file:///src/template.json + parameters="" + + function getOutputs() { + aws cloudformation describe-stacks \ + --stack-name "$STACK_NAME" \ + --query 'Stacks[].Outputs' \ + --output json \ + | jq '.[] | map( { (.OutputKey|tostring): .OutputValue } ) | add' \ + > /outputs.json + } + + # Check if the stack exists + aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>/dev/null || { + if [ -f /src/parameters.json ]; then + parameters="--parameters file:///src/parameters.json" + cat /src/parameters.json + fi + + aws cloudformation create-stack \ + --stack-name "$STACK_NAME" \ + --template-body "file:///src/template.json" \ + --capabilities CAPABILITY_IAM \ + --on-failure "$ON_FAILURE" \ + --timeout-in-minutes "$TIMEOUT" \ + $parameters \ + || { + # Create failed, display errors + aws cloudformation describe-stack-events \ + --stack-name "$STACK_NAME" \ + --max-items 10 \ + | >&2 jq '.StackEvents[] | select((.ResourceStatus | contains("FAILED")) or (.ResourceStatus | contains("ERROR"))) | ("===> ERROR: " + .LogicalResourceId + ": " + .ResourceStatusReason)' + exit 1 + } + + aws cloudformation wait stack-create-complete \ + --stack-name "$STACK_NAME" + + getOutputs + exit 0 + } + + # In case there is an action already in progress, we wait for the corresponding action to complete + wait_action="" + stack_status=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" | jq -r '.Stacks[].StackStatus') + case "$stack_status" in + "CREATE_FAILED") + echo "Deleting previous failed stack..." + aws cloudformation delete-stack --stack-name "$STACK_NAME" + aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || true + ;; + "CREATE_IN_PROGRESS") + echo "Stack create already in progress, waiting..." + aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME" || true + ;; + "UPDATE_IN_PROGRESS") + # Cancel update to avoid stacks stuck in deadlock (re-apply then works) + echo "Stack update already in progress, waiting..." + aws cloudformation cancel-update-stack --stack-name "$STACK_NAME" || true + ;; + "ROLLBACK_IN_PROGRESS") + echo "Stack rollback already in progress, waiting..." + aws cloudformation wait stack-rollback-complete --stack-name "$STACK_NAME" || true + ;; + "DELETE_IN_PROGRESS") + echo "Stack delete already in progress, waiting..." + aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || true + ;; + "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS") + echo "Stack update almost completed, waiting..." + aws cloudformation wait stack-update-complete --stack-name "$STACK_NAME" || true + ;; + esac + + [ -n "$NEVER_UPDATE" ] && { + getOutputs + exit 0 + } + + # Stack exists, trigger an update via `deploy` + if [ -f /src/parameters_overrides.json ]; then + parameters="--parameter-overrides file:///src/parameters_overrides.json" + cat /src/parameters_overrides.json + fi + echo "Deploying stack $STACK_NAME" + aws cloudformation deploy \ + --stack-name "$STACK_NAME" \ + --template-file "/src/template.json" \ + --capabilities CAPABILITY_IAM \ + --no-fail-on-empty-changeset \ + $parameters \ + || { + # Deploy failed, display errors + echo "Failed to deploy stack $STACK_NAME" + aws cloudformation describe-stack-events \ + --stack-name "$STACK_NAME" \ + --max-items 10 \ + | >&2 jq '.StackEvents[] | select((.ResourceStatus | contains("FAILED")) or (.ResourceStatus | contains("ERROR"))) | ("===> ERROR: " + .LogicalResourceId + ": " + .ResourceStatusReason)' + exit 1 + } + + getOutputs + """# From 56bcf6366c997dd363208194034a9e6aa40b8579 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 3 Mar 2021 17:34:00 -0800 Subject: [PATCH 4/5] implemented http-monitoring example using AWS Cloudformation and AWS Cloudwatch Synthetics Signed-off-by: Sam Alba --- examples/aws-monitoring/http_monitor.cue | 269 +++++++++++++++++++++++ examples/aws-monitoring/main.cue | 29 +++ 2 files changed, 298 insertions(+) create mode 100644 examples/aws-monitoring/http_monitor.cue create mode 100644 examples/aws-monitoring/main.cue diff --git a/examples/aws-monitoring/http_monitor.cue b/examples/aws-monitoring/http_monitor.cue new file mode 100644 index 00000000..0840a230 --- /dev/null +++ b/examples/aws-monitoring/http_monitor.cue @@ -0,0 +1,269 @@ +package main + +import ( + "strings" + "regexp" + "encoding/json" + + "dagger.io/aws" + "dagger.io/aws/cloudformation" +) + +#Notification: { + protocol: string + endpoint: string +} + +#Canary: { + name: =~"^[0-9a-z_-]{1,21}$" + slug: strings.Join(regexp.FindAll("[0-9a-zA-Z]*", name, -1), "") + url: string + expectedHTTPCode: *200 | int + timeoutInSeconds: *30 | int + intervalExpression: *"1 minute" | string +} + +#HTTPMonitor: { + + // For sending notifications + notifications: [...#Notification] + // Canaries (tests) + canaries: [...#Canary] + // Name of the Cloudformation stack + cfnStackName: string + // AWS Config + awsConfig: aws.#Config + + cfnStack: cloudformation.#Stack & { + config: awsConfig + source: json.Marshal(#cfnTemplate) + stackName: cfnStackName + onFailure: "DO_NOTHING" + } + + // Function handler + #lambdaHandler: { + url: string + expectedHTTPCode: int + + script: #""" + var synthetics = require('Synthetics'); + const log = require('SyntheticsLogger'); + + const pageLoadBlueprint = async function () { + + // INSERT URL here + const URL = "\#(url)"; + + let page = await synthetics.getPage(); + const response = await page.goto(URL, {waitUntil: 'domcontentloaded', timeout: 30000}); + //Wait for page to render. + //Increase or decrease wait time based on endpoint being monitored. + await page.waitFor(15000); + // This will take a screenshot that will be included in test output artifacts + await synthetics.takeScreenshot('loaded', 'loaded'); + let pageTitle = await page.title(); + log.info('Page title: ' + pageTitle); + if (response.status() !== \#(expectedHTTPCode)) { + throw "Failed to load page!"; + } + }; + + exports.handler = async () => { + return await pageLoadBlueprint(); + }; + """# + } + + #cfnTemplate: { + AWSTemplateFormatVersion: "2010-09-09" + Description: "CloudWatch Synthetics website monitoring" + Resources: { + Topic: { + Type: "AWS::SNS::Topic" + Properties: Subscription: [ + for e in notifications { + Endpoint: e.endpoint + Protocol: e.protocol + }, + ] + } + TopicPolicy: { + Type: "AWS::SNS::TopicPolicy" + Properties: { + PolicyDocument: { + Id: "Id1" + Version: "2012-10-17" + Statement: [ + { + Sid: "Sid1" + Effect: "Allow" + Principal: AWS: "*" + Action: "sns:Publish" + Resource: Ref: "Topic" + Condition: StringEquals: "AWS:SourceOwner": Ref: "AWS::AccountId" + }, + ] + } + Topics: [ + { + Ref: "Topic" + }, + ] + } + } + CanaryBucket: { + Type: "AWS::S3::Bucket" + Properties: {} + } + CanaryRole: { + Type: "AWS::IAM::Role" + Properties: { + AssumeRolePolicyDocument: { + Version: "2012-10-17" + Statement: [ + { + Effect: "Allow" + Principal: Service: "lambda.amazonaws.com" + Action: "sts:AssumeRole" + }, + ] + } + Policies: [ + { + PolicyName: "execution" + PolicyDocument: { + Version: "2012-10-17" + Statement: [ + { + Effect: "Allow" + Action: "s3:ListAllMyBuckets" + Resource: "*" + }, + { + Effect: "Allow" + Action: "s3:PutObject" + Resource: "Fn::Sub": "${CanaryBucket.Arn}/*" + }, + { + Effect: "Allow" + Action: "s3:GetBucketLocation" + Resource: "Fn::GetAtt": [ + "CanaryBucket", + "Arn", + ] + }, + { + Effect: "Allow" + Action: "cloudwatch:PutMetricData" + Resource: "*" + Condition: StringEquals: "cloudwatch:namespace": "CloudWatchSynthetics" + }, + ] + } + }, + ] + } + } + CanaryLogGroup: { + Type: "AWS::Logs::LogGroup" + Properties: { + LogGroupName: "Fn::Sub": "/aws/lambda/cwsyn-\(cfnStackName)" + RetentionInDays: 14 + } + } + CanaryPolicy: { + Type: "AWS::IAM::Policy" + Properties: { + PolicyDocument: Statement: [ + { + Effect: "Allow" + Action: [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + Resource: "Fn::GetAtt": [ + "CanaryLogGroup", + "Arn", + ] + }, + ] + PolicyName: "logs" + Roles: [ + { + Ref: "CanaryRole" + }, + ] + } + } + for canary in canaries { + "Canary\(canary.slug)": { + Type: "AWS::Synthetics::Canary" + Properties: { + ArtifactS3Location: "Fn::Sub": "s3://${CanaryBucket}" + Code: { + #handler: #lambdaHandler & { + url: canary.url + expectedHTTPCode: canary.expectedHTTPCode + } + Handler: "index.handler" + Script: #handler.script + } + ExecutionRoleArn: "Fn::GetAtt": [ + "CanaryRole", + "Arn", + ] + FailureRetentionPeriod: 30 + Name: canary.name + RunConfig: TimeoutInSeconds: canary.timeoutInSeconds + RuntimeVersion: "syn-1.0" + Schedule: { + DurationInSeconds: "0" + Expression: "rate(\(canary.intervalExpression))" + } + StartCanaryAfterCreation: true + SuccessRetentionPeriod: 30 + } + } + "SuccessPercentAlarm\(canary.slug)": { + DependsOn: "TopicPolicy" + Type: "AWS::CloudWatch::Alarm" + Properties: { + AlarmActions: [ + { + Ref: "Topic" + }, + ] + AlarmDescription: "Canary is failing." + ComparisonOperator: "LessThanThreshold" + Dimensions: [ + { + Name: "CanaryName" + Value: Ref: "Canary\(canary.slug)" + }, + ] + EvaluationPeriods: 1 + MetricName: "SuccessPercent" + Namespace: "CloudWatchSynthetics" + OKActions: [ + { + Ref: "Topic" + }, + ] + Period: 300 + Statistic: "Minimum" + Threshold: 90 + TreatMissingData: "notBreaching" + } + } + } + } + Outputs: { + for canary in canaries { + "\(canary.slug)Canary": Value: Ref: "Canary\(canary.slug)" + "\(canary.slug)URL": Value: canary.url + } + NumberCanaries: Value: len(canaries) + } + } +} diff --git a/examples/aws-monitoring/main.cue b/examples/aws-monitoring/main.cue new file mode 100644 index 00000000..57819776 --- /dev/null +++ b/examples/aws-monitoring/main.cue @@ -0,0 +1,29 @@ +package main + +import ( + "dagger.io/aws" +) + +// Fill using: +// --input-string awsConfig.accessKey=XXX +// --input-string awsConfig.secretKey=XXX +awsConfig: aws.#Config & { + region: *"us-east-1" | string +} + +monitor: #HTTPMonitor & { + notifications: [ + #Notification & { + endpoint: "sam+test@blocklayerhq.com" + protocol: "email" + }, + ] + canaries: [ + #Canary & { + name: "website-test" + url: "https://www.google.com/" + }, + ] + cfnStackName: "my-monitor" + "awsConfig": awsConfig +} From bac3ff4f68df05969e0309730a857e076e337ddc Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 3 Mar 2021 18:08:43 -0800 Subject: [PATCH 5/5] stdlib: cuefmt Signed-off-by: Sam Alba --- stdlib/aws/cloudformation/cloudformation.cue | 94 ++++++++++---------- stdlib/dagger/dagger.cue | 8 +- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/stdlib/aws/cloudformation/cloudformation.cue b/stdlib/aws/cloudformation/cloudformation.cue index aa6c2f79..86b89d24 100644 --- a/stdlib/aws/cloudformation/cloudformation.cue +++ b/stdlib/aws/cloudformation/cloudformation.cue @@ -49,53 +49,53 @@ import ( [string]: string #dagger: compute: [ - dagger.#Load & { - from: alpine.#Image & { - package: bash: "=5.1.0-r0" - package: jq: "=1.6-r1" - package: "aws-cli": "=1.18.177-r0" - } - }, - dagger.#Mkdir & { - path: "/src" - }, - for dest, content in #files { - dagger.#WriteFile & { - "dest": dest - "content": content - } - }, - dagger.#Exec & { - args: [ - "/bin/bash", - "--noprofile", - "--norc", - "-eo", - "pipefail", - "/entrypoint.sh", - ] - env: { - AWS_CONFIG_FILE: "/cache/aws/config" - AWS_ACCESS_KEY_ID: config.accessKey - AWS_SECRET_ACCESS_KEY: config.secretKey - AWS_DEFAULT_REGION: config.region - AWS_REGION: config.region - AWS_DEFAULT_OUTPUT: "json" - AWS_PAGER: "" - if neverUpdate { - NEVER_UPDATE: "true" + dagger.#Load & { + from: alpine.#Image & { + package: bash: "=5.1.0-r0" + package: jq: "=1.6-r1" + package: "aws-cli": "=1.18.177-r0" } - STACK_NAME: stackName - TIMEOUT: "\(timeout)" - ON_FAILURE: onFailure - } - dir: "/src" - mount: "/cache/aws": "cache" - }, - dagger.#Export & { - source: "/outputs.json" - format: "json" - }, - ] + }, + dagger.#Mkdir & { + path: "/src" + }, + for dest, content in #files { + dagger.#WriteFile & { + "dest": dest + "content": content + } + }, + dagger.#Exec & { + args: [ + "/bin/bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "/entrypoint.sh", + ] + env: { + AWS_CONFIG_FILE: "/cache/aws/config" + AWS_ACCESS_KEY_ID: config.accessKey + AWS_SECRET_ACCESS_KEY: config.secretKey + AWS_DEFAULT_REGION: config.region + AWS_REGION: config.region + AWS_DEFAULT_OUTPUT: "json" + AWS_PAGER: "" + if neverUpdate { + NEVER_UPDATE: "true" + } + STACK_NAME: stackName + TIMEOUT: "\(timeout)" + ON_FAILURE: onFailure + } + dir: "/src" + mount: "/cache/aws": "cache" + }, + dagger.#Export & { + source: "/outputs.json" + format: "json" + }, + ] } } diff --git a/stdlib/dagger/dagger.cue b/stdlib/dagger/dagger.cue index 3fbcc721..ec86213a 100644 --- a/stdlib/dagger/dagger.cue +++ b/stdlib/dagger/dagger.cue @@ -80,8 +80,8 @@ package dagger } #Mkdir: { - do: "mkdir" - dir: *"/" | string - path: string - mode: int | *0o755 + do: "mkdir" + dir: *"/" | string + path: string + mode: int | *0o755 }