From 5016cf5e30a579d6367383ba4c834c2deca30416 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 19 Jan 2022 17:19:37 -0800 Subject: [PATCH] netlify: Europa port - Fix netlify.#Deploy (there's still FIXMEs) - Externalize the `deploy.sh` script - Add tests - Misc engine fixes for more explicit error messages Signed-off-by: Andrea Luzzardi --- .github/workflows/ci.yml | 13 +++ compiler/build.go | 8 +- pkg/universe.dagger.io/docker/run.cue | 1 + pkg/universe.dagger.io/netlify/deploy.sh | 59 ++++++++++++- pkg/universe.dagger.io/netlify/deploy.sh.cue | 56 ------------ pkg/universe.dagger.io/netlify/netlify.cue | 85 +++++++++++++------ .../netlify/test/netlify-test.cue | 61 +++++++++++++ .../netlify/test/netlify.bats | 9 ++ .../netlify/test/simple/simple.cue | 9 -- pkg/universe.dagger.io/test_secrets.yaml | 21 +++++ plan/plan.go | 4 +- plancontext/fs.go | 4 + plancontext/secret.go | 6 +- plancontext/service.go | 6 +- 14 files changed, 243 insertions(+), 99 deletions(-) mode change 120000 => 100755 pkg/universe.dagger.io/netlify/deploy.sh delete mode 100644 pkg/universe.dagger.io/netlify/deploy.sh.cue create mode 100644 pkg/universe.dagger.io/netlify/test/netlify-test.cue create mode 100644 pkg/universe.dagger.io/netlify/test/netlify.bats delete mode 100644 pkg/universe.dagger.io/netlify/test/simple/simple.cue create mode 100644 pkg/universe.dagger.io/test_secrets.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17bbd197..641918f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,19 @@ jobs: with: go-version: 1.16 + - name: Install Dependencies + run: | + # SOPS + sudo curl -L -o /usr/local/bin/sops https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux + sudo chmod +x /usr/local/bin/sops + + - name: Import AGE Key + env: + DAGGER_AGE_KEY: ${{ secrets.DAGGER_AGE_KEY }} + run: | + mkdir -p ~/.config/sops/age + echo "$DAGGER_AGE_KEY" > ~/.config/sops/age/keys.txt + - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v1 diff --git a/compiler/build.go b/compiler/build.go index 44b0a53b..018c1688 100644 --- a/compiler/build.go +++ b/compiler/build.go @@ -47,7 +47,7 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error return nil }) if err != nil { - return nil, err + return nil, Err(err) } } instances := cueload.Instances(args, buildConfig) @@ -56,16 +56,16 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error } for _, value := range instances { if value.Err != nil { - return nil, value.Err + return nil, Err(value.Err) } } v, err := c.Context.BuildInstances(instances) if err != nil { - return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{})) + return nil, Err(errors.New(cueerrors.Details(err, &cueerrors.Config{}))) } for _, value := range v { if value.Err() != nil { - return nil, value.Err() + return nil, Err(value.Err()) } } if len(v) != 1 { diff --git a/pkg/universe.dagger.io/docker/run.cue b/pkg/universe.dagger.io/docker/run.cue index 3a16efe5..02fd2de9 100644 --- a/pkg/universe.dagger.io/docker/run.cue +++ b/pkg/universe.dagger.io/docker/run.cue @@ -131,6 +131,7 @@ import ( _exec: engine.#Exec & { args: [cmd.name] + cmd._flatFlags + cmd.args input: _image.rootfs + "always": always "mounts": mounts "env": env "workdir": workdir diff --git a/pkg/universe.dagger.io/netlify/deploy.sh b/pkg/universe.dagger.io/netlify/deploy.sh deleted file mode 120000 index 6a5d73ae..00000000 --- a/pkg/universe.dagger.io/netlify/deploy.sh +++ /dev/null @@ -1 +0,0 @@ -deploy.sh.cue \ No newline at end of file diff --git a/pkg/universe.dagger.io/netlify/deploy.sh b/pkg/universe.dagger.io/netlify/deploy.sh new file mode 100755 index 00000000..c7b8d0bd --- /dev/null +++ b/pkg/universe.dagger.io/netlify/deploy.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -e -o pipefail + +NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)" +export NETLIFY_AUTH_TOKEN + +create_site() { + url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" + + curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + -X POST -H "Content-Type: application/json" \ + "$url" \ + -d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body + + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + cat body >&2 + exit 1 + fi + + jq -r '.site_id' body +} + +site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ + "https://api.netlify.com/api/v1/sites?filter=all" | \ + jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \ + ) +if [ -z "$site_id" ] ; then + if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then + echo "Site $NETLIFY_SITE_NAME does not exist" + exit 1 + fi + site_id=$(create_site) + if [ -z "$site_id" ]; then + echo "create site failed" + exit 1 + fi +fi + +netlify link --id "$site_id" +netlify build + +netlify deploy \ + --dir="$(pwd)" \ + --site="$site_id" \ + --prod \ +| tee /tmp/stdout + +url="$(]+' | head -1)" +deployUrl="$(]+' | head -1)" +logsUrl="$(]+' | head -1)" + +# Write output files +mkdir -p /netlify +echo -n "$url" > /netlify/url +echo -n "$deployUrl" > /netlify/deployUrl +echo -n "$logsUrl" > /netlify/logsUrl diff --git a/pkg/universe.dagger.io/netlify/deploy.sh.cue b/pkg/universe.dagger.io/netlify/deploy.sh.cue deleted file mode 100644 index 019dd491..00000000 --- a/pkg/universe.dagger.io/netlify/deploy.sh.cue +++ /dev/null @@ -1,56 +0,0 @@ -package netlify - -_deployScript: #""" - export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)" - - create_site() { - url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites" - - response=$(curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ - -X POST -H "Content-Type: application/json" \ - $url \ - -d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body - ) - if [ $? -ne 0 ]; then - cat body >&2 - exit 1 - fi - - cat body | jq -r '.site_id' - } - - site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ - https://api.netlify.com/api/v1/sites\?filter\=all | \ - jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \ - ) - if [ -z "$site_id" ] ; then - if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then - echo "Site $NETLIFY_SITE_NAME does not exist" - exit 1 - fi - site_id=$(create_site) - if [ -z "$site_id" ]; then - echo "create site failed" - exit 1 - fi - fi - - netlify link --id "$site_id" - netlify build - - netlify deploy \ - --dir="$(pwd)" \ - --site="$site_id" \ - --prod \ - | tee /tmp/stdout - - url=$( /netlify/url - printf "$deployUrl" > /netlify/deployUrl - printf "$logsUrl" > /netlify/logsUrl - """# diff --git a/pkg/universe.dagger.io/netlify/netlify.cue b/pkg/universe.dagger.io/netlify/netlify.cue index 8c94079d..afada701 100644 --- a/pkg/universe.dagger.io/netlify/netlify.cue +++ b/pkg/universe.dagger.io/netlify/netlify.cue @@ -4,10 +4,10 @@ package netlify import ( "dagger.io/dagger" - "universe.dagger.io/docker" + "dagger.io/dagger/engine" "universe.dagger.io/alpine" - "universe.dagger.io/bash" + "universe.dagger.io/docker" ) // Deploy a site to Netlify @@ -35,30 +35,61 @@ import ( // Create the site if it doesn't exist create: *true | false - // Execute `netlify deploy` in a container - command: bash.#Run & { - // Container image. `netlify` must be available in the execution path - *{ - _buildDefaultImage: docker.#Build & { - input: alpine.#Build & { - bash: version: "=~5.1" - jq: version: "=~1.6" + // Source code of the Netlify package + _source: engine.#Source & { + path: "." + include: ["*.sh"] + } + + _build: docker.#Build & { + steps: [ + alpine.#Build & { + packages: { + bash: {} curl: {} - yarn: version: "=~1.22" + jq: {} + yarn: {} } - steps: [{ - run: script: "yarn global add netlify-cli@3.38.10" - }] - } + }, + docker.#Run & { + cmd: { + name: "yarn" + args: ["global", "add", "netlify-cli@8.6.21"] + } + }, + docker.#Copy & { + contents: _source.output + dest: "/app" + }, + ] + } - // No nested tasks, boo hoo hoo - image: _buildDefaultImage.output - env: CUSTOM_IMAGE: "0" - } | { - env: CUSTOM_IMAGE: "1" - } + // Execute `netlify deploy` in a container + command: docker.#Run & { + // FIXME: custom base image not supported + // Container image. `netlify` must be available in the execution path + // *{ + // _buildDefaultImage: docker.#Build & { + // input: alpine.#Build & { + // bash: version: "=~5.1" + // jq: version: "=~1.6" + // curl: {} + // yarn: version: "=~1.22" + // } + // steps: [{ + // run: script: "yarn global add netlify-cli@3.38.10" + // }] + // } + + // // No nested tasks, boo hoo hoo + // image: _buildDefaultImage.output + // env: CUSTOM_IMAGE: "0" + // } | { + // env: CUSTOM_IMAGE: "1" + // } + + image: _build.output - script: _deployScript // see deploy.sh always: true env: { NETLIFY_SITE_NAME: site @@ -81,7 +112,9 @@ import ( contents: token } } - output: files: { + cmd: name: "/app/deploy.sh" + + export: files: { "/netlify/url": _ "/netlify/deployUrl": _ "/netlify/logsUrl": _ @@ -89,11 +122,11 @@ import ( } // URL of the deployed site - url: command.output.files."/netlify/url".contents + url: command.export.files."/netlify/url".contents // URL of the latest deployment - deployUrl: command.output.files."/netlify/deployUrl".contents + deployUrl: command.export.files."/netlify/deployUrl".contents // URL for logs of the latest deployment - logsUrl: command.output.files."/netlify/logsUrl".contents + logsUrl: command.export.files."/netlify/logsUrl".contents } diff --git a/pkg/universe.dagger.io/netlify/test/netlify-test.cue b/pkg/universe.dagger.io/netlify/test/netlify-test.cue new file mode 100644 index 00000000..3507c969 --- /dev/null +++ b/pkg/universe.dagger.io/netlify/test/netlify-test.cue @@ -0,0 +1,61 @@ +package yarn + +import ( + "encoding/yaml" + + "dagger.io/dagger" + "dagger.io/dagger/engine" + + "universe.dagger.io/netlify" + "universe.dagger.io/alpine" + "universe.dagger.io/bash" +) + +dagger.#Plan & { + inputs: secrets: test: command: { + name: "sops" + args: ["-d", "../../test_secrets.yaml"] + } + + actions: { + testSecrets: engine.#TransformSecret & { + input: inputs.secrets.test.contents + #function: { + input: _ + output: yaml.Unmarshal(input) + } + } + + marker: "hello world" + + data: engine.#WriteFile & { + input: engine.#Scratch + path: "index.html" + contents: marker + } + + // Deploy to netlify + deploy: netlify.#Deploy & { + team: "blocklayer" + token: testSecrets.output.netlifyToken.contents + + site: "dagger-test" + contents: data.output + } + + _alpine: alpine.#Build & { + packages: { + bash: {} + curl: {} + } + } + + // Check if the website was deployed + verify: bash.#Run & { + input: _alpine.output + script: #""" + test "$(curl \#(deploy.deployUrl))" = "\#(marker)" + """# + } + } +} diff --git a/pkg/universe.dagger.io/netlify/test/netlify.bats b/pkg/universe.dagger.io/netlify/test/netlify.bats new file mode 100644 index 00000000..9f665e9e --- /dev/null +++ b/pkg/universe.dagger.io/netlify/test/netlify.bats @@ -0,0 +1,9 @@ +setup() { + load '../../bats_helpers' + + common_setup +} + +@test "netlify.#Deploy" { + dagger up ./netlify-test.cue +} diff --git a/pkg/universe.dagger.io/netlify/test/simple/simple.cue b/pkg/universe.dagger.io/netlify/test/simple/simple.cue deleted file mode 100644 index e9e78744..00000000 --- a/pkg/universe.dagger.io/netlify/test/simple/simple.cue +++ /dev/null @@ -1,9 +0,0 @@ -package netlify - -import ( - "dagger.io/dagger" -) - -deploy: #Deploy & { - contents: dagger.#Scratch -} diff --git a/pkg/universe.dagger.io/test_secrets.yaml b/pkg/universe.dagger.io/test_secrets.yaml new file mode 100644 index 00000000..68f4c664 --- /dev/null +++ b/pkg/universe.dagger.io/test_secrets.yaml @@ -0,0 +1,21 @@ +netlifyToken: ENC[AES256_GCM,data:DeTBgf73iiIDVJZ3i1Rd6Cn9KvJGwh7n8/u/zWKdpaMvU7R1X43JqMbZMg==,iv:0HmdJr7BHKQk+RrCWAzZCkU7BkJ5N5//otgwAgJnQ6w=,tag:DoVYsCnO6HMHXpakX4uBlA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUEhWbjV3M29oUUJyWk81 + Wk1WQ1E0cmtuVlhNSGxkWUM3WmJXdUYvbzAwCjlFWW9IVmtmTjY1aU1LR2lxWFlT + am9RemNqSDRWK2FDYk1xeGNiTFlWMFUKLS0tIFVrSzBCMERQbnhYb09ReVpFK00v + TG5YUDlFVzlRRFBCdEhsNVlVK1dMRTgKx1TPZWWQiaU8iMni03/ekG+m4rFCcaa4 + JI+ED2d+8411BgZtlss/ukQtwskidvYTvetyWw2jes6o1lhfDv5q2A== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2022-01-20T00:42:44Z" + mac: ENC[AES256_GCM,data:N4dbowNmz34Hn/o1Ofv4g9Z5I7EzcYyrGpXSu9fkczd69zkTpv87uFamEdV/kQM2bbIEm9gS8d0oTi41qsC0iax368YUJmjG6xMptwrrA/mcjRzwXjlPrCZN9454srJw4NXWm0F5/aJQa4XlO65OCLZw+4WCz0wyAWwKzuQNAb0=,iv:EIG55jdEIbVp390uCVJ/rCjJO+s+CsAblH0/CIMNgIc=,tag:dcZDoMsBToikTQ83R0azag==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.1 diff --git a/plan/plan.go b/plan/plan.go index 7e8e0d1e..25cd27bc 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -111,7 +111,9 @@ func (p *Plan) configPlatform() error { // prepare executes the pre-run hooks of tasks func (p *Plan) prepare(ctx context.Context) error { flow := cueflow.New( - &cueflow.Config{}, + &cueflow.Config{ + FindHiddenTasks: true, + }, p.source.Cue(), func(flowVal cue.Value) (cueflow.Runner, error) { v := compiler.Wrap(flowVal) diff --git a/plancontext/fs.go b/plancontext/fs.go index f1213ce3..70b3ec0d 100644 --- a/plancontext/fs.go +++ b/plancontext/fs.go @@ -77,6 +77,10 @@ func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) { c.l.RLock() defer c.l.RUnlock() + if !v.LookupPath(fsIDPath).IsConcrete() { + return nil, fmt.Errorf("invalid FS at path %q: FS is not set", v.Path()) + } + // This is #Scratch, so we'll return an empty FS if v.LookupPath(fsIDPath).Kind() == cue.NullKind { return &FS{}, nil diff --git a/plancontext/secret.go b/plancontext/secret.go index 7b72126e..c0ab648c 100644 --- a/plancontext/secret.go +++ b/plancontext/secret.go @@ -64,9 +64,13 @@ func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) { c.l.RLock() defer c.l.RUnlock() + if !v.LookupPath(secretIDPath).IsConcrete() { + return nil, fmt.Errorf("invalid secret at path %q: secret is not set", v.Path()) + } + id, err := v.LookupPath(secretIDPath).String() if err != nil { - return nil, fmt.Errorf("invalid secret %q: %w", v.Path(), err) + return nil, fmt.Errorf("invalid secret at path %q: %w", v.Path(), err) } secret, ok := c.store[id] diff --git a/plancontext/service.go b/plancontext/service.go index 11f43ef6..2cc2a85b 100644 --- a/plancontext/service.go +++ b/plancontext/service.go @@ -71,9 +71,13 @@ func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) { c.l.RLock() defer c.l.RUnlock() + if !v.LookupPath(serviceIDPath).IsConcrete() { + return nil, fmt.Errorf("invalid service at path %q: service is not set", v.Path()) + } + id, err := v.LookupPath(serviceIDPath).String() if err != nil { - return nil, fmt.Errorf("invalid service %q: %w", v.Path(), err) + return nil, fmt.Errorf("invalid service at path %q: %w", v.Path(), err) } s, ok := c.store[id]