Merge pull request #153 from dagger/example-aws-monitoring

AWS Cloudformation + HTTP monitoring example

dagger/compiler/value.go

@@ -92,6 +92,11 @@ func (v *Value) String() (string, error) {
 	return v.val.String()
 }
 
+// Proxy function to the underlying cue.Value
+func (v *Value) Int64() (int64, error) {
+	return v.val.Int64()
+}
+
 func (v *Value) SourceUnsafe() string {
 	s, _ := v.SourceString()
 	return s

dagger/pipeline.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"strings"
 
 	"github.com/docker/distribution/reference"
@@ -167,6 +168,10 @@ func (p *Pipeline) doOp(ctx context.Context, op *compiler.Value) error {
 		return p.Subdir(ctx, op)
 	case "docker-build":
 		return p.DockerBuild(ctx, op)
+	case "write-file":
+		return p.WriteFile(ctx, op)
+	case "mkdir":
+		return p.Mkdir(ctx, op)
 	default:
 		return fmt.Errorf("invalid operation: %s", op.JSON())
 	}
@@ -679,3 +684,55 @@ func (p *Pipeline) DockerBuild(ctx context.Context, op *compiler.Value) error {
 
 	return nil
 }
+
+func (p *Pipeline) WriteFile(ctx context.Context, op *compiler.Value) error {
+	content, err := op.Get("content").String()
+	if err != nil {
+		return err
+	}
+
+	dest, err := op.Get("dest").String()
+	if err != nil {
+		return err
+	}
+
+	mode, err := op.Get("mode").Int64()
+	if err != nil {
+		return err
+	}
+
+	p.fs = p.fs.Change(func(st llb.State) llb.State {
+		return st.File(
+			llb.Mkfile(dest, fs.FileMode(mode), []byte(content)),
+			llb.WithCustomName(p.vertexNamef("WriteFile %s", dest)),
+		)
+	})
+
+	return nil
+}
+
+func (p *Pipeline) Mkdir(ctx context.Context, op *compiler.Value) error {
+	path, err := op.Get("path").String()
+	if err != nil {
+		return err
+	}
+
+	dir, err := op.Get("dir").String()
+	if err != nil {
+		return err
+	}
+
+	mode, err := op.Get("mode").Int64()
+	if err != nil {
+		return err
+	}
+
+	p.fs = p.fs.Change(func(st llb.State) llb.State {
+		return st.Dir(dir).File(
+			llb.Mkdir(path, fs.FileMode(mode)),
+			llb.WithCustomName(p.vertexNamef("Mkdir %s", path)),
+		)
+	})
+
+	return nil
+}

examples/aws-monitoring/http_monitor.cue

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"strings"
+	"regexp"
+	"encoding/json"
+
+	"dagger.io/aws"
+	"dagger.io/aws/cloudformation"
+)
+
+#Notification: {
+	protocol: string
+	endpoint: string
+}
+
+#Canary: {
+	name:               =~"^[0-9a-z_-]{1,21}$"
+	slug:               strings.Join(regexp.FindAll("[0-9a-zA-Z]*", name, -1), "")
+	url:                string
+	expectedHTTPCode:   *200 | int
+	timeoutInSeconds:   *30 | int
+	intervalExpression: *"1 minute" | string
+}
+
+#HTTPMonitor: {
+
+	// For sending notifications
+	notifications: [...#Notification]
+	// Canaries (tests)
+	canaries: [...#Canary]
+	// Name of the Cloudformation stack
+	cfnStackName: string
+	// AWS Config
+	awsConfig: aws.#Config
+
+	cfnStack: cloudformation.#Stack & {
+		config:    awsConfig
+		source:    json.Marshal(#cfnTemplate)
+		stackName: cfnStackName
+		onFailure: "DO_NOTHING"
+	}
+
+	// Function handler
+	#lambdaHandler: {
+		url:              string
+		expectedHTTPCode: int
+
+		script: #"""
+			var synthetics = require('Synthetics');
+			const log = require('SyntheticsLogger');
+
+			const pageLoadBlueprint = async function () {
+
+				// INSERT URL here
+				const URL = "\#(url)";
+
+				let page = await synthetics.getPage();
+				const response = await page.goto(URL, {waitUntil: 'domcontentloaded', timeout: 30000});
+				//Wait for page to render.
+				//Increase or decrease wait time based on endpoint being monitored.
+				await page.waitFor(15000);
+				// This will take a screenshot that will be included in test output artifacts
+				await synthetics.takeScreenshot('loaded', 'loaded');
+				let pageTitle = await page.title();
+				log.info('Page title: ' + pageTitle);
+				if (response.status() !== \#(expectedHTTPCode)) {
+					throw "Failed to load page!";
+				}
+			};
+
+			exports.handler = async () => {
+				return await pageLoadBlueprint();
+			};
+			"""#
+	}
+
+	#cfnTemplate: {
+		AWSTemplateFormatVersion: "2010-09-09"
+		Description:              "CloudWatch Synthetics website monitoring"
+		Resources: {
+			Topic: {
+				Type: "AWS::SNS::Topic"
+				Properties: Subscription: [
+					for e in notifications {
+						Endpoint: e.endpoint
+						Protocol: e.protocol
+					},
+				]
+			}
+			TopicPolicy: {
+				Type: "AWS::SNS::TopicPolicy"
+				Properties: {
+					PolicyDocument: {
+						Id:      "Id1"
+						Version: "2012-10-17"
+						Statement: [
+							{
+								Sid:    "Sid1"
+								Effect: "Allow"
+								Principal: AWS: "*"
+								Action: "sns:Publish"
+								Resource: Ref: "Topic"
+								Condition: StringEquals: "AWS:SourceOwner": Ref: "AWS::AccountId"
+							},
+						]
+					}
+					Topics: [
+						{
+							Ref: "Topic"
+						},
+					]
+				}
+			}
+			CanaryBucket: {
+				Type: "AWS::S3::Bucket"
+				Properties: {}
+			}
+			CanaryRole: {
+				Type: "AWS::IAM::Role"
+				Properties: {
+					AssumeRolePolicyDocument: {
+						Version: "2012-10-17"
+						Statement: [
+							{
+								Effect: "Allow"
+								Principal: Service: "lambda.amazonaws.com"
+								Action: "sts:AssumeRole"
+							},
+						]
+					}
+					Policies: [
+						{
+							PolicyName: "execution"
+							PolicyDocument: {
+								Version: "2012-10-17"
+								Statement: [
+									{
+										Effect:   "Allow"
+										Action:   "s3:ListAllMyBuckets"
+										Resource: "*"
+									},
+									{
+										Effect: "Allow"
+										Action: "s3:PutObject"
+										Resource: "Fn::Sub": "${CanaryBucket.Arn}/*"
+									},
+									{
+										Effect: "Allow"
+										Action: "s3:GetBucketLocation"
+										Resource: "Fn::GetAtt": [
+											"CanaryBucket",
+											"Arn",
+										]
+									},
+									{
+										Effect:   "Allow"
+										Action:   "cloudwatch:PutMetricData"
+										Resource: "*"
+										Condition: StringEquals: "cloudwatch:namespace": "CloudWatchSynthetics"
+									},
+								]
+							}
+						},
+					]
+				}
+			}
+			CanaryLogGroup: {
+				Type: "AWS::Logs::LogGroup"
+				Properties: {
+					LogGroupName: "Fn::Sub": "/aws/lambda/cwsyn-\(cfnStackName)"
+					RetentionInDays: 14
+				}
+			}
+			CanaryPolicy: {
+				Type: "AWS::IAM::Policy"
+				Properties: {
+					PolicyDocument: Statement: [
+						{
+							Effect: "Allow"
+							Action: [
+								"logs:CreateLogStream",
+								"logs:PutLogEvents",
+							]
+							Resource: "Fn::GetAtt": [
+								"CanaryLogGroup",
+								"Arn",
+							]
+						},
+					]
+					PolicyName: "logs"
+					Roles: [
+						{
+							Ref: "CanaryRole"
+						},
+					]
+				}
+			}
+			for canary in canaries {
+				"Canary\(canary.slug)": {
+					Type: "AWS::Synthetics::Canary"
+					Properties: {
+						ArtifactS3Location: "Fn::Sub": "s3://${CanaryBucket}"
+						Code: {
+							#handler: #lambdaHandler & {
+								url:              canary.url
+								expectedHTTPCode: canary.expectedHTTPCode
+							}
+							Handler: "index.handler"
+							Script:  #handler.script
+						}
+						ExecutionRoleArn: "Fn::GetAtt": [
+							"CanaryRole",
+							"Arn",
+						]
+						FailureRetentionPeriod: 30
+						Name:                   canary.name
+						RunConfig: TimeoutInSeconds: canary.timeoutInSeconds
+						RuntimeVersion: "syn-1.0"
+						Schedule: {
+							DurationInSeconds: "0"
+							Expression:        "rate(\(canary.intervalExpression))"
+						}
+						StartCanaryAfterCreation: true
+						SuccessRetentionPeriod:   30
+					}
+				}
+				"SuccessPercentAlarm\(canary.slug)": {
+					DependsOn: "TopicPolicy"
+					Type:      "AWS::CloudWatch::Alarm"
+					Properties: {
+						AlarmActions: [
+							{
+								Ref: "Topic"
+							},
+						]
+						AlarmDescription:   "Canary is failing."
+						ComparisonOperator: "LessThanThreshold"
+						Dimensions: [
+							{
+								Name: "CanaryName"
+								Value: Ref: "Canary\(canary.slug)"
+							},
+						]
+						EvaluationPeriods: 1
+						MetricName:        "SuccessPercent"
+						Namespace:         "CloudWatchSynthetics"
+						OKActions: [
+							{
+								Ref: "Topic"
+							},
+						]
+						Period:           300
+						Statistic:        "Minimum"
+						Threshold:        90
+						TreatMissingData: "notBreaching"
+					}
+				}
+			}
+		}
+		Outputs: {
+			for canary in canaries {
+				"\(canary.slug)Canary": Value: Ref: "Canary\(canary.slug)"
+				"\(canary.slug)URL": Value: canary.url
+			}
+			NumberCanaries: Value: len(canaries)
+		}
+	}
+}

examples/aws-monitoring/main.cue

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"dagger.io/aws"
+)
+
+// Fill using:
+//          --input-string awsConfig.accessKey=XXX
+//          --input-string awsConfig.secretKey=XXX
+awsConfig: aws.#Config & {
+	region: *"us-east-1" | string
+}
+
+monitor: #HTTPMonitor & {
+	notifications: [
+		#Notification & {
+			endpoint: "sam+test@blocklayerhq.com"
+			protocol: "email"
+		},
+	]
+	canaries: [
+		#Canary & {
+			name: "website-test"
+			url:  "https://www.google.com/"
+		},
+	]
+	cfnStackName: "my-monitor"
+	"awsConfig":  awsConfig
+}

stdlib/aws/aws.cue

@@ -0,0 +1,10 @@
+package aws
+
+#Config: {
+	// AWS region
+	region: string
+	// AWS access key
+	accessKey: string // FIXME: should be a secret
+	// AWS secret key
+	secretKey: string // FIXME: should be a secret
+}

stdlib/aws/cloudformation/cloudformation.cue

@@ -0,0 +1,101 @@
+package cloudformation
+
+import (
+	"encoding/json"
+
+	"dagger.io/dagger"
+	"dagger.io/alpine"
+	"dagger.io/aws"
+)
+
+// AWS CloudFormation Stack
+#Stack: {
+
+	// AWS Config
+	config: aws.#Config
+
+	// Source is the Cloudformation template (JSON/YAML string)
+	source: string
+
+	// Stackname is the cloudformation stack
+	stackName: string
+
+	// Stack parameters
+	parameters: [string]: _
+
+	// Behavior when failure to create/update the Stack
+	onFailure: *"DO_NOTHING" | "ROLLBACK" | "DELETE"
+
+	// Timeout for waiting for the stack to be created/updated (in minutes)
+	timeout: *10 | uint
+
+	// Never update the stack if already exists
+	neverUpdate: *false | bool
+
+	#files: {
+		"/entrypoint.sh":     #Code
+		"/src/template.json": source
+		if len(parameters) > 0 {
+			"/src/parameters.json": json.Marshal(
+						[ for key, val in parameters {
+									ParameterKey:   "\(key)"
+									ParameterValue: "\(val)"
+								}])
+			"/src/parameters_overrides.json": json.Marshal([ for key, val in parameters {"\(key)=\(val)"}])
+		}
+	}
+
+	outputs: {
+		[string]: string
+
+		#dagger: compute: [
+			dagger.#Load & {
+				from: alpine.#Image & {
+					package: bash:      "=5.1.0-r0"
+					package: jq:        "=1.6-r1"
+					package: "aws-cli": "=1.18.177-r0"
+				}
+			},
+			dagger.#Mkdir & {
+				path: "/src"
+			},
+			for dest, content in #files {
+				dagger.#WriteFile & {
+					"dest":    dest
+					"content": content
+				}
+			},
+			dagger.#Exec & {
+				args: [
+					"/bin/bash",
+					"--noprofile",
+					"--norc",
+					"-eo",
+					"pipefail",
+					"/entrypoint.sh",
+				]
+				env: {
+					AWS_CONFIG_FILE:       "/cache/aws/config"
+					AWS_ACCESS_KEY_ID:     config.accessKey
+					AWS_SECRET_ACCESS_KEY: config.secretKey
+					AWS_DEFAULT_REGION:    config.region
+					AWS_REGION:            config.region
+					AWS_DEFAULT_OUTPUT:    "json"
+					AWS_PAGER:             ""
+					if neverUpdate {
+						NEVER_UPDATE: "true"
+					}
+					STACK_NAME: stackName
+					TIMEOUT:    "\(timeout)"
+					ON_FAILURE: onFailure
+				}
+				dir: "/src"
+				mount: "/cache/aws": "cache"
+			},
+			dagger.#Export & {
+				source: "/outputs.json"
+				format: "json"
+			},
+		]
+	}
+}

stdlib/aws/cloudformation/code.cue

@@ -0,0 +1,108 @@
+package cloudformation
+
+#Code: #"""
+	set +o pipefail
+
+	aws cloudformation validate-template --template-body file:///src/template.json
+	parameters=""
+
+	function getOutputs() {
+	    aws cloudformation describe-stacks \
+	        --stack-name "$STACK_NAME" \
+	        --query 'Stacks[].Outputs' \
+	        --output json \
+	        | jq '.[] | map( { (.OutputKey|tostring): .OutputValue } ) | add' \
+	        > /outputs.json
+	}
+
+	# Check if the stack exists
+	aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>/dev/null || {
+	    if [ -f /src/parameters.json ]; then
+	        parameters="--parameters file:///src/parameters.json"
+	        cat /src/parameters.json
+	    fi
+
+	    aws cloudformation create-stack \
+	        --stack-name "$STACK_NAME" \
+	        --template-body "file:///src/template.json" \
+	        --capabilities CAPABILITY_IAM \
+	        --on-failure "$ON_FAILURE" \
+	        --timeout-in-minutes "$TIMEOUT" \
+	        $parameters \
+	    || {
+	        # Create failed, display errors
+	        aws cloudformation describe-stack-events \
+	            --stack-name "$STACK_NAME" \
+	            --max-items 10 \
+	            | >&2 jq '.StackEvents[] | select((.ResourceStatus | contains("FAILED")) or (.ResourceStatus | contains("ERROR"))) | ("===> ERROR: " + .LogicalResourceId + ": " + .ResourceStatusReason)'
+	        exit 1
+	    }
+
+	    aws cloudformation wait stack-create-complete \
+	        --stack-name "$STACK_NAME"
+
+	    getOutputs
+	    exit 0
+	}
+
+	# In case there is an action already in progress, we wait for the corresponding action to complete
+	wait_action=""
+	stack_status=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" | jq -r '.Stacks[].StackStatus')
+	case "$stack_status" in
+	    "CREATE_FAILED")
+	        echo "Deleting previous failed stack..."
+	        aws cloudformation delete-stack --stack-name "$STACK_NAME"
+	        aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || true
+	        ;;
+	    "CREATE_IN_PROGRESS")
+	        echo "Stack create already in progress, waiting..."
+	        aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME" || true
+	        ;;
+	    "UPDATE_IN_PROGRESS")
+	        # Cancel update to avoid stacks stuck in deadlock (re-apply then works)
+	        echo "Stack update already in progress, waiting..."
+	        aws cloudformation cancel-update-stack --stack-name "$STACK_NAME" || true
+	        ;;
+	    "ROLLBACK_IN_PROGRESS")
+	        echo "Stack rollback already in progress, waiting..."
+	        aws cloudformation wait stack-rollback-complete --stack-name "$STACK_NAME" || true
+	        ;;
+	    "DELETE_IN_PROGRESS")
+	        echo "Stack delete already in progress, waiting..."
+	        aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || true
+	        ;;
+	    "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS")
+	        echo "Stack update almost completed, waiting..."
+	        aws cloudformation wait stack-update-complete --stack-name "$STACK_NAME" || true
+	        ;;
+	esac
+
+	[ -n "$NEVER_UPDATE" ] && {
+	    getOutputs
+	    exit 0
+	}
+
+	# Stack exists, trigger an update via `deploy`
+	if [ -f /src/parameters_overrides.json ]; then
+	    parameters="--parameter-overrides file:///src/parameters_overrides.json"
+	    cat /src/parameters_overrides.json
+	fi
+	echo "Deploying stack $STACK_NAME"
+	aws cloudformation deploy \
+	    --stack-name "$STACK_NAME" \
+	    --template-file "/src/template.json" \
+	    --capabilities CAPABILITY_IAM \
+	    --no-fail-on-empty-changeset \
+	    $parameters \
+	|| {
+	    # Deploy failed, display errors
+	    echo "Failed to deploy stack $STACK_NAME"
+	    aws cloudformation describe-stack-events \
+	        --stack-name "$STACK_NAME" \
+	        --max-items 10 \
+	        | >&2 jq '.StackEvents[] | select((.ResourceStatus | contains("FAILED")) or (.ResourceStatus | contains("ERROR"))) | ("===> ERROR: " + .LogicalResourceId + ": " + .ResourceStatusReason)'
+	    exit 1
+	}
+
+	getOutputs
+	"""#

stdlib/dagger/dagger.cue

@@ -71,3 +71,17 @@ package dagger
 	buildArg?: [string]: string
 	label?: [string]:    string
 }
+
+#WriteFile: {
+	do:      "write-file"
+	content: string
+	dest:    string
+	mode:    int | *0o644
+}
+
+#Mkdir: {
+	do:   "mkdir"
+	dir:  *"/" | string
+	path: string
+	mode: int | *0o755
+}

stdlib/stdlib.go

@@ -11,7 +11,7 @@ import (
 )
 
 // FS contains the filesystem of the stdlib.
-//go:embed **/*.cue
+//go:embed **/*.cue **/*/*.cue
 var FS embed.FS
 
 const (