netlify: Europa port

- Fix netlify.#Deploy (there's still FIXMEs)
- Externalize the `deploy.sh` script
- Add tests
- Misc engine fixes for more explicit error messages

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>
This commit is contained in:
Andrea Luzzardi 2022-01-19 17:19:37 -08:00
parent 7cd17c39bc
commit 5016cf5e30
14 changed files with 243 additions and 99 deletions

View File

@ -181,6 +181,19 @@ jobs:
with: with:
go-version: 1.16 go-version: 1.16
- name: Install Dependencies
run: |
# SOPS
sudo curl -L -o /usr/local/bin/sops https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux
sudo chmod +x /usr/local/bin/sops
- name: Import AGE Key
env:
DAGGER_AGE_KEY: ${{ secrets.DAGGER_AGE_KEY }}
run: |
mkdir -p ~/.config/sops/age
echo "$DAGGER_AGE_KEY" > ~/.config/sops/age/keys.txt
- name: Expose GitHub Runtime - name: Expose GitHub Runtime
uses: crazy-max/ghaction-github-runtime@v1 uses: crazy-max/ghaction-github-runtime@v1

View File

@ -47,7 +47,7 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error
return nil return nil
}) })
if err != nil { if err != nil {
return nil, err return nil, Err(err)
} }
} }
instances := cueload.Instances(args, buildConfig) instances := cueload.Instances(args, buildConfig)
@ -56,16 +56,16 @@ func Build(src string, overlays map[string]fs.FS, args ...string) (*Value, error
} }
for _, value := range instances { for _, value := range instances {
if value.Err != nil { if value.Err != nil {
return nil, value.Err return nil, Err(value.Err)
} }
} }
v, err := c.Context.BuildInstances(instances) v, err := c.Context.BuildInstances(instances)
if err != nil { if err != nil {
return nil, errors.New(cueerrors.Details(err, &cueerrors.Config{})) return nil, Err(errors.New(cueerrors.Details(err, &cueerrors.Config{})))
} }
for _, value := range v { for _, value := range v {
if value.Err() != nil { if value.Err() != nil {
return nil, value.Err() return nil, Err(value.Err())
} }
} }
if len(v) != 1 { if len(v) != 1 {

View File

@ -131,6 +131,7 @@ import (
_exec: engine.#Exec & { _exec: engine.#Exec & {
args: [cmd.name] + cmd._flatFlags + cmd.args args: [cmd.name] + cmd._flatFlags + cmd.args
input: _image.rootfs input: _image.rootfs
"always": always
"mounts": mounts "mounts": mounts
"env": env "env": env
"workdir": workdir "workdir": workdir

View File

@ -1 +0,0 @@
deploy.sh.cue

View File

@ -0,0 +1,58 @@
#!/bin/bash
set -e -o pipefail
NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)"
export NETLIFY_AUTH_TOKEN
create_site() {
url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-X POST -H "Content-Type: application/json" \
"$url" \
-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body
# shellcheck disable=SC2181
if [ $? -ne 0 ]; then
cat body >&2
exit 1
fi
jq -r '.site_id' body
}
site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
"https://api.netlify.com/api/v1/sites?filter=all" | \
jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
)
if [ -z "$site_id" ] ; then
if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
echo "Site $NETLIFY_SITE_NAME does not exist"
exit 1
fi
site_id=$(create_site)
if [ -z "$site_id" ]; then
echo "create site failed"
exit 1
fi
fi
netlify link --id "$site_id"
netlify build
netlify deploy \
--dir="$(pwd)" \
--site="$site_id" \
--prod \
| tee /tmp/stdout
url="$(</tmp/stdout grep Website | grep -Eo 'https://[^ >]+' | head -1)"
deployUrl="$(</tmp/stdout grep Unique | grep -Eo 'https://[^ >]+' | head -1)"
logsUrl="$(</tmp/stdout grep Logs | grep -Eo 'https://[^ >]+' | head -1)"
# Write output files
mkdir -p /netlify
echo -n "$url" > /netlify/url
echo -n "$deployUrl" > /netlify/deployUrl
echo -n "$logsUrl" > /netlify/logsUrl

View File

@ -1,56 +0,0 @@
package netlify
_deployScript: #"""
export NETLIFY_AUTH_TOKEN="$(cat /run/secrets/token)"
create_site() {
url="https://api.netlify.com/api/v1/${NETLIFY_ACCOUNT:-}/sites"
response=$(curl -s -S --fail-with-body -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
-X POST -H "Content-Type: application/json" \
$url \
-d "{\"name\": \"${NETLIFY_SITE_NAME}\", \"custom_domain\": \"${NETLIFY_DOMAIN}\"}" -o body
)
if [ $? -ne 0 ]; then
cat body >&2
exit 1
fi
cat body | jq -r '.site_id'
}
site_id=$(curl -s -S -f -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \
https://api.netlify.com/api/v1/sites\?filter\=all | \
jq -r ".[] | select(.name==\"$NETLIFY_SITE_NAME\") | .id" \
)
if [ -z "$site_id" ] ; then
if [ "${NETLIFY_SITE_CREATE:-}" != 1 ]; then
echo "Site $NETLIFY_SITE_NAME does not exist"
exit 1
fi
site_id=$(create_site)
if [ -z "$site_id" ]; then
echo "create site failed"
exit 1
fi
fi
netlify link --id "$site_id"
netlify build
netlify deploy \
--dir="$(pwd)" \
--site="$site_id" \
--prod \
| tee /tmp/stdout
url=$(</tmp/stdout sed -n -e 's/^Website URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
deployUrl=$(</tmp/stdout sed -n -e 's/^Unique Deploy URL:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
logsUrl=$(</tmp/stdout sed -n -e 's/^Logs:.*\(https:\/\/.*\)$/\1/p' | tr -d '\n')
# Write output files
mkdir -p /netlify
printf "$url" > /netlify/url
printf "$deployUrl" > /netlify/deployUrl
printf "$logsUrl" > /netlify/logsUrl
"""#

View File

@ -4,10 +4,10 @@ package netlify
import ( import (
"dagger.io/dagger" "dagger.io/dagger"
"universe.dagger.io/docker" "dagger.io/dagger/engine"
"universe.dagger.io/alpine" "universe.dagger.io/alpine"
"universe.dagger.io/bash" "universe.dagger.io/docker"
) )
// Deploy a site to Netlify // Deploy a site to Netlify
@ -35,30 +35,61 @@ import (
// Create the site if it doesn't exist // Create the site if it doesn't exist
create: *true | false create: *true | false
// Execute `netlify deploy` in a container // Source code of the Netlify package
command: bash.#Run & { _source: engine.#Source & {
// Container image. `netlify` must be available in the execution path path: "."
*{ include: ["*.sh"]
_buildDefaultImage: docker.#Build & { }
input: alpine.#Build & {
bash: version: "=~5.1" _build: docker.#Build & {
jq: version: "=~1.6" steps: [
alpine.#Build & {
packages: {
bash: {}
curl: {} curl: {}
yarn: version: "=~1.22" jq: {}
yarn: {}
} }
steps: [{ },
run: script: "yarn global add netlify-cli@3.38.10" docker.#Run & {
}] cmd: {
name: "yarn"
args: ["global", "add", "netlify-cli@8.6.21"]
}
},
docker.#Copy & {
contents: _source.output
dest: "/app"
},
]
} }
// No nested tasks, boo hoo hoo // Execute `netlify deploy` in a container
image: _buildDefaultImage.output command: docker.#Run & {
env: CUSTOM_IMAGE: "0" // FIXME: custom base image not supported
} | { // Container image. `netlify` must be available in the execution path
env: CUSTOM_IMAGE: "1" // *{
} // _buildDefaultImage: docker.#Build & {
// input: alpine.#Build & {
// bash: version: "=~5.1"
// jq: version: "=~1.6"
// curl: {}
// yarn: version: "=~1.22"
// }
// steps: [{
// run: script: "yarn global add netlify-cli@3.38.10"
// }]
// }
// // No nested tasks, boo hoo hoo
// image: _buildDefaultImage.output
// env: CUSTOM_IMAGE: "0"
// } | {
// env: CUSTOM_IMAGE: "1"
// }
image: _build.output
script: _deployScript // see deploy.sh
always: true always: true
env: { env: {
NETLIFY_SITE_NAME: site NETLIFY_SITE_NAME: site
@ -81,7 +112,9 @@ import (
contents: token contents: token
} }
} }
output: files: { cmd: name: "/app/deploy.sh"
export: files: {
"/netlify/url": _ "/netlify/url": _
"/netlify/deployUrl": _ "/netlify/deployUrl": _
"/netlify/logsUrl": _ "/netlify/logsUrl": _
@ -89,11 +122,11 @@ import (
} }
// URL of the deployed site // URL of the deployed site
url: command.output.files."/netlify/url".contents url: command.export.files."/netlify/url".contents
// URL of the latest deployment // URL of the latest deployment
deployUrl: command.output.files."/netlify/deployUrl".contents deployUrl: command.export.files."/netlify/deployUrl".contents
// URL for logs of the latest deployment // URL for logs of the latest deployment
logsUrl: command.output.files."/netlify/logsUrl".contents logsUrl: command.export.files."/netlify/logsUrl".contents
} }

View File

@ -0,0 +1,61 @@
package yarn
import (
"encoding/yaml"
"dagger.io/dagger"
"dagger.io/dagger/engine"
"universe.dagger.io/netlify"
"universe.dagger.io/alpine"
"universe.dagger.io/bash"
)
dagger.#Plan & {
inputs: secrets: test: command: {
name: "sops"
args: ["-d", "../../test_secrets.yaml"]
}
actions: {
testSecrets: engine.#TransformSecret & {
input: inputs.secrets.test.contents
#function: {
input: _
output: yaml.Unmarshal(input)
}
}
marker: "hello world"
data: engine.#WriteFile & {
input: engine.#Scratch
path: "index.html"
contents: marker
}
// Deploy to netlify
deploy: netlify.#Deploy & {
team: "blocklayer"
token: testSecrets.output.netlifyToken.contents
site: "dagger-test"
contents: data.output
}
_alpine: alpine.#Build & {
packages: {
bash: {}
curl: {}
}
}
// Check if the website was deployed
verify: bash.#Run & {
input: _alpine.output
script: #"""
test "$(curl \#(deploy.deployUrl))" = "\#(marker)"
"""#
}
}
}

View File

@ -0,0 +1,9 @@
setup() {
load '../../bats_helpers'
common_setup
}
@test "netlify.#Deploy" {
dagger up ./netlify-test.cue
}

View File

@ -1,9 +0,0 @@
package netlify
import (
"dagger.io/dagger"
)
deploy: #Deploy & {
contents: dagger.#Scratch
}

View File

@ -0,0 +1,21 @@
netlifyToken: ENC[AES256_GCM,data:DeTBgf73iiIDVJZ3i1Rd6Cn9KvJGwh7n8/u/zWKdpaMvU7R1X43JqMbZMg==,iv:0HmdJr7BHKQk+RrCWAzZCkU7BkJ5N5//otgwAgJnQ6w=,tag:DoVYsCnO6HMHXpakX4uBlA==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age:
- recipient: age1gxwmtwahzwdmrskhf90ppwlnze30lgpm056kuesrxzeuyclrwvpsupwtpk
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnUEhWbjV3M29oUUJyWk81
Wk1WQ1E0cmtuVlhNSGxkWUM3WmJXdUYvbzAwCjlFWW9IVmtmTjY1aU1LR2lxWFlT
am9RemNqSDRWK2FDYk1xeGNiTFlWMFUKLS0tIFVrSzBCMERQbnhYb09ReVpFK00v
TG5YUDlFVzlRRFBCdEhsNVlVK1dMRTgKx1TPZWWQiaU8iMni03/ekG+m4rFCcaa4
JI+ED2d+8411BgZtlss/ukQtwskidvYTvetyWw2jes6o1lhfDv5q2A==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2022-01-20T00:42:44Z"
mac: ENC[AES256_GCM,data:N4dbowNmz34Hn/o1Ofv4g9Z5I7EzcYyrGpXSu9fkczd69zkTpv87uFamEdV/kQM2bbIEm9gS8d0oTi41qsC0iax368YUJmjG6xMptwrrA/mcjRzwXjlPrCZN9454srJw4NXWm0F5/aJQa4XlO65OCLZw+4WCz0wyAWwKzuQNAb0=,iv:EIG55jdEIbVp390uCVJ/rCjJO+s+CsAblH0/CIMNgIc=,tag:dcZDoMsBToikTQ83R0azag==,type:str]
pgp: []
unencrypted_suffix: _unencrypted
version: 3.7.1

View File

@ -111,7 +111,9 @@ func (p *Plan) configPlatform() error {
// prepare executes the pre-run hooks of tasks // prepare executes the pre-run hooks of tasks
func (p *Plan) prepare(ctx context.Context) error { func (p *Plan) prepare(ctx context.Context) error {
flow := cueflow.New( flow := cueflow.New(
&cueflow.Config{}, &cueflow.Config{
FindHiddenTasks: true,
},
p.source.Cue(), p.source.Cue(),
func(flowVal cue.Value) (cueflow.Runner, error) { func(flowVal cue.Value) (cueflow.Runner, error) {
v := compiler.Wrap(flowVal) v := compiler.Wrap(flowVal)

View File

@ -77,6 +77,10 @@ func (c *fsContext) FromValue(v *compiler.Value) (*FS, error) {
c.l.RLock() c.l.RLock()
defer c.l.RUnlock() defer c.l.RUnlock()
if !v.LookupPath(fsIDPath).IsConcrete() {
return nil, fmt.Errorf("invalid FS at path %q: FS is not set", v.Path())
}
// This is #Scratch, so we'll return an empty FS // This is #Scratch, so we'll return an empty FS
if v.LookupPath(fsIDPath).Kind() == cue.NullKind { if v.LookupPath(fsIDPath).Kind() == cue.NullKind {
return &FS{}, nil return &FS{}, nil

View File

@ -64,9 +64,13 @@ func (c *secretContext) FromValue(v *compiler.Value) (*Secret, error) {
c.l.RLock() c.l.RLock()
defer c.l.RUnlock() defer c.l.RUnlock()
if !v.LookupPath(secretIDPath).IsConcrete() {
return nil, fmt.Errorf("invalid secret at path %q: secret is not set", v.Path())
}
id, err := v.LookupPath(secretIDPath).String() id, err := v.LookupPath(secretIDPath).String()
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid secret %q: %w", v.Path(), err) return nil, fmt.Errorf("invalid secret at path %q: %w", v.Path(), err)
} }
secret, ok := c.store[id] secret, ok := c.store[id]

View File

@ -71,9 +71,13 @@ func (c *serviceContext) FromValue(v *compiler.Value) (*Service, error) {
c.l.RLock() c.l.RLock()
defer c.l.RUnlock() defer c.l.RUnlock()
if !v.LookupPath(serviceIDPath).IsConcrete() {
return nil, fmt.Errorf("invalid service at path %q: service is not set", v.Path())
}
id, err := v.LookupPath(serviceIDPath).String() id, err := v.LookupPath(serviceIDPath).String()
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid service %q: %w", v.Path(), err) return nil, fmt.Errorf("invalid service at path %q: %w", v.Path(), err)
} }
s, ok := c.store[id] s, ok := c.store[id]