netlify: Europa port

- Fix netlify.#Deploy (there's still FIXMEs)
- Externalize the `deploy.sh` script
- Add tests
- Misc engine fixes for more explicit error messages

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

.github/workflows/ci.yml

@@ -181,6 +181,19 @@ jobs:
         with:
           go-version: 1.16
 
+      - name: Install Dependencies
+        run: |
+          # SOPS
+          sudo curl -L -o /usr/local/bin/sops https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux
+          sudo chmod +x /usr/local/bin/sops
+
+      - name: Import AGE Key
+        env:
+          DAGGER_AGE_KEY: ${{ secrets.DAGGER_AGE_KEY }}
+        run: |
+          mkdir -p ~/.config/sops/age
+          echo "$DAGGER_AGE_KEY" > ~/.config/sops/age/keys.txt
+
       - name: Expose GitHub Runtime
         uses: crazy-max/ghaction-github-runtime@v1
 

compiler/build.go

@@ -47,7 +47,7 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error
 			return nil
 		})
 		if err != nil {
-			return nil, err
+			return nil, Err(err)
 		}
 	}
 	instances := cueload.Instances(args, buildConfig)
@@ -56,16 +56,16 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error
 	}
 	for _, value := range instances {
 		if value.Err != nil {
-			return nil, value.Err
+			return nil, Err(value.Err)
 		}
 	}
 	v, err := c.Context.BuildInstances(instances)
 	if err != nil {
-		return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{}))
+		return nil, Err(errors.New(cueerrors.Details(err, &cueerrors.Config{})))
 	}
 	for _, value := range v {
 		if value.Err() != nil {
-			return nil, value.Err()
+			return nil, Err(value.Err())
 		}
 	}
 	if len(v) != 1 {

pkg/universe.dagger.io/docker/run.cue

@@ -131,6 +131,7 @@ import (
 	_exec: engine.#Exec & {
 		args:      [cmd.name] + cmd._flatFlags + cmd.args
 		input:     _image.rootfs
+		"always":  always
 		"mounts":  mounts
 		"env":     env
 		"workdir": workdir

pkg/universe.dagger.io/netlify/deploy.sh

@@ -1 +0,0 @@
-deploy.sh.cue

pkg/universe.dagger.io/netlify/deploy.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)"
+export NETLIFY_AUTH_TOKEN
+
+create_site() {
+	url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
+
+	curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
+		-X POST -H "Content-Type: application/json" \
+		"$url" \
+		-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body
+
+	# shellcheck disable=SC2181
+	if [ $? -ne 0 ]; then
+		cat body >&2
+		exit 1
+	fi
+
+	jq -r '.site_id' body
+}
+
+site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
+		"https://api.netlify.com/api/v1/sites?filter=all" | \
+		jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
+	)
+if [ -z "$site_id" ] ; then
+	if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
+		echo "Site $NETLIFY_SITE_NAME does not exist"
+		exit 1
+	fi
+	site_id=$(create_site)
+	if [ -z "$site_id" ]; then
+		echo "create site failed"
+		exit 1
+	fi
+fi
+
+netlify link --id "$site_id"
+netlify build
+
+netlify deploy \
+	--dir="$(pwd)" \
+	--site="$site_id" \
+	--prod \
+| tee /tmp/stdout
+
+url="$(</tmp/stdout grep Website | grep -Eo 'https://[^ >]+' | head -1)"
+deployUrl="$(</tmp/stdout grep Unique | grep -Eo 'https://[^ >]+' | head -1)"
+logsUrl="$(</tmp/stdout grep Logs | grep -Eo 'https://[^ >]+' | head -1)"
+
+# Write output files
+mkdir -p /netlify
+echo -n "$url" > /netlify/url
+echo -n "$deployUrl" > /netlify/deployUrl
+echo -n "$logsUrl" > /netlify/logsUrl

pkg/universe.dagger.io/netlify/deploy.sh.cue

@@ -1,56 +0,0 @@
-package netlify
-
-_deployScript: #"""
-	export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)"
-	
-	create_site() {
-	    url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
-	
-	    response=$(curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-			-X POST -H "Content-Type: application/json" \
-			$url \
-			-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body
-		    )
-	    if [ $? -ne 0 ]; then
-		cat body >&2
-		exit 1
-	    fi
-	
-	    cat body | jq -r '.site_id' 
-	}
-	
-	site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-		    https://api.netlify.com/api/v1/sites\?filter\=all | \
-		    jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
-		)
-	if [ -z "$site_id" ] ; then
-	    if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
-		echo "Site $NETLIFY_SITE_NAME does not exist"
-		exit 1
-	    fi
-	    site_id=$(create_site)
-	    if [ -z "$site_id" ]; then
-		echo "create site failed"
-		exit 1
-	    fi
-	fi
-	
-	netlify link --id "$site_id"
-	netlify build
-	
-	netlify deploy \
-	    --dir="$(pwd)" \
-	    --site="$site_id" \
-	    --prod \
-	| tee /tmp/stdout
-	
-	url=$(</tmp/stdout sed -n -e 's/^Website URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
-	deployUrl=$(</tmp/stdout sed -n -e 's/^Unique Deploy URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
-	logsUrl=$(</tmp/stdout sed -n -e 's/^Logs:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
-	
-	# Write output files
-	mkdir -p /netlify
-	printf "$url" > /netlify/url
-	printf "$deployUrl" > /netlify/deployUrl
-	printf "$logsUrl" > /netlify/logsUrl
-	"""#

pkg/universe.dagger.io/netlify/netlify.cue

@@ -4,10 +4,10 @@ package netlify
 
 import (
 	"dagger.io/dagger"
-	"universe.dagger.io/docker"
+	"dagger.io/dagger/engine"
 
 	"universe.dagger.io/alpine"
-	"universe.dagger.io/bash"
+	"universe.dagger.io/docker"
 )
 
 // Deploy a site to Netlify
@@ -35,30 +35,61 @@ import (
 	// Create the site if it doesn't exist
 	create: *true | false
 
-	// Execute `netlify deploy` in a container
+	// Source code of the Netlify package
-	command: bash.#Run & {
+	_source: engine.#Source & {
-		// Container image. `netlify` must be available in the execution path
+		path: "."
-		*{
+		include: ["*.sh"]
-			_buildDefaultImage: docker.#Build & {
+	}
-				input: alpine.#Build & {
+
-					bash: version: "=~5.1"
+	_build: docker.#Build & {
-					jq: version:   "=~1.6"
+		steps: [
+			alpine.#Build & {
+				packages: {
+					bash: {}
 					curl: {}
-					yarn: version: "=~1.22"
+					jq: {}
+					yarn: {}
 				}
-				steps: [{
+			},
-					run: script: "yarn global add netlify-cli@3.38.10"
+			docker.#Run & {
-				}]
+				cmd: {
+					name: "yarn"
+					args: ["global", "add", "netlify-cli@8.6.21"]
+				}
+			},
+			docker.#Copy & {
+				contents: _source.output
+				dest:     "/app"
+			},
+		]
 	}
 
-			// No nested tasks, boo hoo hoo
+	// Execute `netlify deploy` in a container
-			image: _buildDefaultImage.output
+	command: docker.#Run & {
-			env: CUSTOM_IMAGE: "0"
+		// FIXME: custom base image not supported
-		} | {
+		// Container image. `netlify` must be available in the execution path
-			env: CUSTOM_IMAGE: "1"
+		// *{
-		}
+		//  _buildDefaultImage: docker.#Build & {
+		//   input: alpine.#Build & {
+		//    bash: version: "=~5.1"
+		//    jq: version:   "=~1.6"
+		//    curl: {}
+		//    yarn: version: "=~1.22"
+		//   }
+		//   steps: [{
+		//    run: script: "yarn global add netlify-cli@3.38.10"
+		//   }]
+		//  }
+
+		//  // No nested tasks, boo hoo hoo
+		//  image: _buildDefaultImage.output
+		//  env: CUSTOM_IMAGE: "0"
+		// } | {
+		//  env: CUSTOM_IMAGE: "1"
+		// }
+
+		image: _build.output
 
-		script: _deployScript // see deploy.sh
 		always: true
 		env: {
 			NETLIFY_SITE_NAME: site
@@ -81,7 +112,9 @@ import (
 				contents: token
 			}
 		}
-		output: files: {
+		cmd: name: "/app/deploy.sh"
+
+		export: files: {
 			"/netlify/url":       _
 			"/netlify/deployUrl": _
 			"/netlify/logsUrl":   _
@@ -89,11 +122,11 @@ import (
 	}
 
 	// URL of the deployed site
-	url: command.output.files."/netlify/url".contents
+	url: command.export.files."/netlify/url".contents
 
 	// URL of the latest deployment
-	deployUrl: command.output.files."/netlify/deployUrl".contents
+	deployUrl: command.export.files."/netlify/deployUrl".contents
 
 	// URL for logs of the latest deployment
-	logsUrl: command.output.files."/netlify/logsUrl".contents
+	logsUrl: command.export.files."/netlify/logsUrl".contents
 }

pkg/universe.dagger.io/netlify/test/netlify-test.cue

@@ -0,0 +1,61 @@
+package yarn
+
+import (
+	"encoding/yaml"
+
+	"dagger.io/dagger"
+	"dagger.io/dagger/engine"
+
+	"universe.dagger.io/netlify"
+	"universe.dagger.io/alpine"
+	"universe.dagger.io/bash"
+)
+
+dagger.#Plan & {
+	inputs: secrets: test: command: {
+		name: "sops"
+		args: ["-d", "../../test_secrets.yaml"]
+	}
+
+	actions: {
+		testSecrets: engine.#TransformSecret & {
+			input: inputs.secrets.test.contents
+			#function: {
+				input:  _
+				output: yaml.Unmarshal(input)
+			}
+		}
+
+		marker: "hello world"
+
+		data: engine.#WriteFile & {
+			input:    engine.#Scratch
+			path:     "index.html"
+			contents: marker
+		}
+
+		// Deploy to netlify
+		deploy: netlify.#Deploy & {
+			team:  "blocklayer"
+			token: testSecrets.output.netlifyToken.contents
+
+			site:     "dagger-test"
+			contents: data.output
+		}
+
+		_alpine: alpine.#Build & {
+			packages: {
+				bash: {}
+				curl: {}
+			}
+		}
+
+		// Check if the website was deployed
+		verify: bash.#Run & {
+			input:  _alpine.output
+			script: #"""
+		  test "$(curl \#(deploy.deployUrl))" = "\#(marker)"
+		  """#
+		}
+	}
+}

pkg/universe.dagger.io/netlify/test/netlify.bats

@@ -0,0 +1,9 @@
+setup() {
+    load '../../bats_helpers'
+
+    common_setup
+}
+
+@test "netlify.#Deploy" {
+    dagger up ./netlify-test.cue
+}

pkg/universe.dagger.io/netlify/test/simple/simple.cue

@@ -1,9 +0,0 @@
-package netlify
-
-import (
-	"dagger.io/dagger"
-)
-
-deploy: #Deploy & {
-	contents: dagger.#Scratch
-}

pkg/universe.dagger.io/test_secrets.yaml

@@ -0,0 +1,21 @@
+netlifyToken: ENC[AES256_GCM,data:DeTBgf73iiIDVJZ3i1Rd6Cn9KvJGwh7n8/u/zWKdpaMvU7R1X43JqMbZMg==,iv:0HmdJr7BHKQk+RrCWAzZCkU7BkJ5N5//otgwAgJnQ6w=,tag:DoVYsCnO6HMHXpakX4uBlA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUEhWbjV3M29oUUJyWk81
+            Wk1WQ1E0cmtuVlhNSGxkWUM3WmJXdUYvbzAwCjlFWW9IVmtmTjY1aU1LR2lxWFlT
+            am9RemNqSDRWK2FDYk1xeGNiTFlWMFUKLS0tIFVrSzBCMERQbnhYb09ReVpFK00v
+            TG5YUDlFVzlRRFBCdEhsNVlVK1dMRTgKx1TPZWWQiaU8iMni03/ekG+m4rFCcaa4
+            JI+ED2d+8411BgZtlss/ukQtwskidvYTvetyWw2jes6o1lhfDv5q2A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-01-20T00:42:44Z"
+    mac: ENC[AES256_GCM,data:N4dbowNmz34Hn/o1Ofv4g9Z5I7EzcYyrGpXSu9fkczd69zkTpv87uFamEdV/kQM2bbIEm9gS8d0oTi41qsC0iax368YUJmjG6xMptwrrA/mcjRzwXjlPrCZN9454srJw4NXWm0F5/aJQa4XlO65OCLZw+4WCz0wyAWwKzuQNAb0=,iv:EIG55jdEIbVp390uCVJ/rCjJO+s+CsAblH0/CIMNgIc=,tag:dcZDoMsBToikTQ83R0azag==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

plan/plan.go

@@ -111,7 +111,9 @@ func (p *Plan) configPlatform() error {
 // prepare executes the pre-run hooks of tasks
 func (p *Plan) prepare(ctx context.Context) error {
 	flow := cueflow.New(
-		&cueflow.Config{},
+		&cueflow.Config{
+			FindHiddenTasks: true,
+		},
 		p.source.Cue(),
 		func(flowVal cue.Value) (cueflow.Runner, error) {
 			v := compiler.Wrap(flowVal)

plancontext/fs.go

@@ -77,6 +77,10 @@ func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) {
 	c.l.RLock()
 	defer c.l.RUnlock()
 
+	if !v.LookupPath(fsIDPath).IsConcrete() {
+		return nil, fmt.Errorf("invalid FS at path %q: FS is not set", v.Path())
+	}
+
 	// This is #Scratch, so we'll return an empty FS
 	if v.LookupPath(fsIDPath).Kind() == cue.NullKind {
 		return &FS{}, nil

plancontext/secret.go

@@ -64,9 +64,13 @@ func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) {
 	c.l.RLock()
 	defer c.l.RUnlock()
 
+	if !v.LookupPath(secretIDPath).IsConcrete() {
+		return nil, fmt.Errorf("invalid secret at path %q: secret is not set", v.Path())
+	}
+
 	id, err := v.LookupPath(secretIDPath).String()
 	if err != nil {
-		return nil, fmt.Errorf("invalid secret %q: %w", v.Path(), err)
+		return nil, fmt.Errorf("invalid secret at path %q: %w", v.Path(), err)
 	}
 
 	secret, ok := c.store[id]

plancontext/service.go

@@ -71,9 +71,13 @@ func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) {
 	c.l.RLock()
 	defer c.l.RUnlock()
 
+	if !v.LookupPath(serviceIDPath).IsConcrete() {
+		return nil, fmt.Errorf("invalid service at path %q: service is not set", v.Path())
+	}
+
 	id, err := v.LookupPath(serviceIDPath).String()
 	if err != nil {
-		return nil, fmt.Errorf("invalid service %q: %w", v.Path(), err)
+		return nil, fmt.Errorf("invalid service at path %q: %w", v.Path(), err)
 	}
 
 	s, ok := c.store[id]