Merge pull request #26 from dubo-dubon-duponey/testing

Adding tests

.github/workflows/ci.yml

@@ -17,8 +17,10 @@ jobs:
           go-version: 1.14.2
         id: go
 
-      - name: Insatall Dependencies
+      - name: Install Dependencies
         run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends shellcheck
           curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
           curl -L https://github.com/cuelang/cue/releases/download/v0.3.0-alpha6/cue_0.3.0-alpha6_Linux_x86_64.tar.gz | sudo tar zxf - -C /usr/local/bin
 
@@ -33,6 +35,14 @@ jobs:
         run: |
           make lint
 
+      - name: Start buildkit
+        run: |
+          docker run -d --name buildkitd --privileged moby/buildkit:v0.8.1@sha256:ecd5ad4910c322cad6995f8a1a0805d9da4b09ed4aaef40627f5bcb8ebf74068
+
+      - name: Integration test
+        run: |
+          make integration
+
       - name: Test
         run: |
           make test

Makefile

@@ -22,3 +22,10 @@ lint: generate cuefmt
 	golangci-lint run
 	@test -z "$$(git status -s . | grep -e "^ M"  | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)"
 	@test -z "$$(git status -s . | grep -e "^ M"  | grep gen.go | cut -d ' ' -f3 | tee /dev/stderr)"
+
+.PHONY: integration
+integration: dagger
+	# Self-diagnostics
+	./examples/tests/test-test.sh 2>/dev/null
+	# Actual integration tests
+	./examples/tests/test.sh all

examples/tests/README.md

@@ -0,0 +1,20 @@
+# Testing
+
+## TL;DR
+
+```
+# Get help
+./test.sh --help
+
+# Run all tests
+# You can also just call ./test.sh with no argument
+# Also, `make integration` does exactly that
+./test.sh all
+
+# Run one random dagger cue directory, with expectations on exit code, stdout, stderr
+./test.sh fetch-git/nonexistent/ref --exit=1 --stdout=
+```
+
+By default, the dagger binary is expected to be found in ../../cmd/dagger/dagger relative to the test.sh script.
+
+If you need to change this, pass along `DAGGER_BINARY=somewhere/dagger`

examples/tests/compute/invalid/bool/main.cue

@@ -0,0 +1,5 @@
+package testing
+
+#dagger: {
+	compute: true
+}

examples/tests/compute/invalid/int/main.cue

@@ -0,0 +1,5 @@
+package testing
+
+#dagger: {
+	compute: 123
+}

examples/tests/compute/invalid/string/main.cue

@@ -0,0 +1,5 @@
+package testing
+
+#dagger: {
+	compute: "whatever"
+}

examples/tests/compute/invalid/struct/main.cue

@@ -0,0 +1,7 @@
+package testing
+
+#dagger: {
+	compute: {
+		whatever: "wrong"
+	}
+}

examples/tests/compute/invalid/undefined_prop/main.cue

@@ -0,0 +1,18 @@
+package testing
+
+bar: string
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-container"
+			ref: "alpine"
+		},
+		{
+			do: "exec"
+			dir: "/"
+			args: ["sh", "-c", "echo \(foo.bar)"]
+		}
+	]
+	foo: bar: bar
+}

examples/tests/compute/noop/main.cue

@@ -0,0 +1,19 @@
+package testing
+
+// no-op, should not error
+realempty: {
+	#dagger: {}
+}
+
+// no-op, should not error
+empty: {
+	#dagger: compute: []
+}
+
+// additional prop, should not error
+withprops: {
+	#dagger: {
+		compute: []
+		foo: bar: "foo"
+	}
+}

examples/tests/compute/simple/main.cue

@@ -0,0 +1,15 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-container"
+			ref: "alpine"
+		},
+		{
+			do: "exec"
+			args: ["true"]
+			dir: "/"
+		}
+	]
+}

examples/tests/exec/always/main.cue

@@ -0,0 +1,15 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["echo", "always output"]
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+		always: true
+	},
+]

examples/tests/exec/dir/doesnotexist/main.cue

@@ -0,0 +1,13 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", "echo should not succeed"]
+		dir: "/thisisnonexistent"
+	},
+]

examples/tests/exec/dir/exist/main.cue

@@ -0,0 +1,16 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", """
+			echo "pwd is: $(pwd)"
+			[ "$(pwd)" == "/etc" ] || exit 1
+		"""]
+		dir: "/etc"
+	},
+]

examples/tests/exec/env/invalid/main.cue

@@ -0,0 +1,17 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", #"""
+		echo "$foo"
+		"""#]
+		env: foo: {lala: "lala"}
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+]

examples/tests/exec/env/overlay/main.cue

@@ -0,0 +1,21 @@
+package testing
+
+bar: string
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", """
+			echo "foo: $foo"
+			[ "$foo" == "overlay environment" ] || exit 1
+		"""]
+		env: foo: bar
+		always: true
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+]

examples/tests/exec/env/valid/main.cue

@@ -0,0 +1,17 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["sh", "-c", """
+		[ "$foo" == "output environment" ] || exit 1
+		"""]
+		env: foo: "output environment"
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+]

examples/tests/exec/error/main.cue

@@ -0,0 +1,14 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["erroringout"]
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+]

examples/tests/exec/invalid/main.cue

@@ -0,0 +1,13 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+}

examples/tests/exec/simple/main.cue

@@ -0,0 +1,14 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine"
+	},
+	{
+		do: "exec"
+		args: ["echo", "simple output"]
+		// XXX Blocked by https://github.com/blocklayerhq/dagger/issues/19
+		dir: "/"
+	},
+]

examples/tests/fetch-container/exist/main.cue

@@ -0,0 +1,48 @@
+package testing
+
+// XXX https://github.com/blocklayerhq/dagger/issues/10 requires that #dagger are nested under https://github.com/blocklayerhq/dagger/issues/21 makes this very hard to verify
+
+busybox1: {
+	#dagger: compute: [
+		{
+			do: "fetch-container"
+			ref: "busybox"
+		},
+	]
+}
+
+busybox2: {
+	#dagger: compute: [
+		{
+			do: "fetch-container"
+			ref: "busybox:latest"
+		},
+	]
+}
+
+busybox3: {
+	#dagger: compute: [
+		{
+			do: "fetch-container"
+			ref: "busybox:1.33-musl"
+		},
+	]
+}
+
+busybox4: {
+	#dagger: compute: [
+		{
+			do: "fetch-container"
+			ref: "busyboxa@sha256:e2af53705b841ace3ab3a44998663d4251d33ee8a9acaf71b66df4ae01c3bbe7"
+		},
+	]
+}
+
+busybox5: {
+	#dagger: compute: [
+		{
+			do: "fetch-container"
+			ref: "busybox:1.33-musl@sha256:e2af53705b841ace3ab3a44998663d4251d33ee8a9acaf71b66df4ae01c3bbe7"
+		},
+	]
+}

examples/tests/fetch-container/invalid/main.cue

@@ -0,0 +1,7 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+	},
+]

examples/tests/fetch-container/nonexistent/digest/main.cue

@@ -0,0 +1,8 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine@sha256:c6c7524e2111f22a9f7577211232d89a9e68cf5b9ed4a41ba77957c9771380a5"
+	}
+]

examples/tests/fetch-container/nonexistent/image-with-valid-digest/main.cue

@@ -0,0 +1,10 @@
+package testing
+
+// XXX WATCHOUT
+// Once buildkit has pulled that digest, it will stay cached and happily succeed WHATEVER the image name then is
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "busyboxaaa@sha256:e2af53705b841ace3ab3a44998663d4251d33ee8a9acaf71b66df4ae01c3bbe7"
+	},
+]

examples/tests/fetch-container/nonexistent/image/main.cue

@@ -0,0 +1,8 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "doesnotexist"
+	}
+]

examples/tests/fetch-container/nonexistent/tag/main.cue

@@ -0,0 +1,8 @@
+package testing
+
+#dagger: compute: [
+	{
+		do: "fetch-container"
+		ref: "alpine:doesnotexist"
+	}
+]

examples/tests/fetch-git/exist/main.cue

@@ -0,0 +1,11 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-git"
+			remote: "https://github.com/blocklayerhq/acme-clothing.git"
+			ref: "master"
+		}
+	]
+}

examples/tests/fetch-git/invalid/main.cue

@@ -0,0 +1,9 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-git"
+		}
+	]
+}

examples/tests/fetch-git/nonexistent/bork/main.cue

@@ -0,0 +1,11 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-git"
+			remote: "pork://pork"
+			ref: "master"
+		}
+	]
+}

examples/tests/fetch-git/nonexistent/ref/main.cue

@@ -0,0 +1,11 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-git"
+			remote: "https://github.com/blocklayerhq/acme-clothing.git"
+			ref: "lalalalal"
+		}
+	]
+}

examples/tests/fetch-git/nonexistent/remote/main.cue

@@ -0,0 +1,11 @@
+package testing
+
+#dagger: {
+	compute: [
+		{
+			do: "fetch-git"
+			remote: "https://github.com/blocklayerhq/lalalala.git"
+			ref: "master"
+		}
+	]
+}

examples/tests/test-lib.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+
+########################################################################
+# Logging helpers
+########################################################################
+readonly COLOR_RED=1
+readonly COLOR_GREEN=2
+readonly COLOR_YELLOW=3
+
+# Prefix a date to a log line and output to stderr
+logger::stamp(){
+  local color="$1"
+  local level="$2"
+  local i
+  shift
+  shift
+  [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput setaf "$color"
+  for i in "$@"; do
+    >&2 printf "[%s] [%s] %s\\n" "$(date)" "$level" "$i"
+  done
+  [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op
+}
+
+logger::info(){
+  logger::stamp "$COLOR_GREEN" "INFO" "$@"
+}
+
+logger::warning(){
+  logger::stamp "$COLOR_YELLOW" "WARNING" "$@"
+}
+
+logger::error(){
+  logger::stamp "$COLOR_RED" "ERROR" "$@"
+}
+
+########################################################################
+# Handle exit code and errors of random command
+########################################################################
+wrap::err(){
+  local args=()
+  local actualExit=0
+  local expectedExit=0
+  local ret=0
+
+  for i in "$@"; do
+    if [ "${i:0:9}" == "--stderr=" ]; then
+      local expectedStderr="${i:9}"
+    elif [ "${i:0:7}" == "--exit=" ]; then
+      expectedExit="${i:7}"
+      expectedExit="${expectedExit:-0}"
+    else
+      args+=("$i")
+    fi
+  done
+
+  logger::info " -> ${args[*]}"
+
+  exec 3>&1
+  actualStderr="$("${args[@]}" 2>&1 1>&3)" || actualExit="$?"
+
+  [ "$expectedExit" == "$actualExit" ] || {
+    logger::error " -> Expected exit code: $expectedExit" " -> Actual exit code  : $actualExit"
+    logger::error " -> Stderr was:"
+    >&2 jq <<<"$actualStderr" 2>/dev/null || {
+      >&2 echo "$actualStderr"
+      logger::error " -> Also, stderr is not json"
+    }
+    ret=1
+  }
+
+  [ -z "${expectedStderr+x}" ] || [ "$expectedStderr" == "$actualStderr" ] || {
+    logger::error " -> Expected stderr:"
+    >&2 jq <<<"$expectedStderr" 2>/dev/null || {
+      >&2 echo "$expectedStderr"
+    }
+    logger::error " -> Actual stderr  :"
+    >&2 jq <<<"$actualStderr" 2>/dev/null || {
+      >&2 echo "$actualStderr"
+      logger::error " -> Also, stderr is not json ^"
+    }
+    ret=1
+  }
+
+  exec 3>&-
+  return "$ret"
+}
+
+########################################################################
+# Main test function
+#    argument 1 is a test description
+#    to test the exit code, pass --exit=int (if not provided, will test that the command exits succesfully)
+#    to test the value of stderr, pass --stderr=string (if not provided, stderr is not verified)
+#    to test the value of stdout, pass --stdout=string (if not provided, stdout is not verified)
+#    any other argument is the command that is going to be run
+# Example:
+#    test dagger compute somecue --exit=1 --stderr=expectederror
+########################################################################
+test::one(){
+  local testDescription="$1"
+  shift
+  local args=()
+  local ret=0
+
+  for i in "$@"; do
+    if [ "${i:0:9}" == "--stdout=" ]; then
+      local expectedStdout="${i:9}"
+    else
+      args+=("$i")
+    fi
+  done
+
+  logger::info "$testDescription"
+
+  local actualStdout
+  actualStdout="$(wrap::err "${args[@]}")" || {
+    ret=1
+  }
+
+  [ -z "${expectedStdout+x}" ] || [ "$expectedStdout" == "$actualStdout" ] || {
+    exec 3>&-
+    logger::error " -> Expected stdout: $expectedStdout" " -> Actual stdout  : $actualStdout"
+    ret=1
+  }
+
+  [ "$ret" != 0 ] || logger::info " -> Success"
+  return "$ret"
+}

examples/tests/test-test.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+
+# The purpose of this is solely to test::one the test::one framework in test-lib.sh
+
+readonly d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
+
+# Performing self-diagnostic linting check first
+shellcheck "$d/"*.sh
+
+# shellcheck source=/dev/null
+. "$d/test-lib.sh"
+
+########################################################################
+# Verifying the test::one framework is working
+########################################################################
+self::command(){
+  local stdout="${1:-}"
+  local stderr="${2:-}"
+  local ret="${3:-0}"
+  printf "%s" "$stdout"
+  >&2 printf "%s" "$stderr"
+  exit "$ret"
+}
+
+self::test(){
+  # Command success testing
+  test::one "Command success, no expectation should succeed" self::command || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, --exit=0 should succeed" self::command --exit=0 || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, --exit=1 should fail" self::command --exit=1 && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, matching --stderr should succeed" self::command "" "to err" --stderr="to err" || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, non matching --stderr should fail" self::command --stderr="to stderr" && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, matching --stdout foo should succeed" self::command "lol foo" --stdout="lol foo" || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, non matching --stdout should fail" self::command "lol foo" --stdout="lol" && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command success, all expectation match should succeed" self::command "lol" --exit=0 --stdout="lol" --stderr= || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  # Command failure testing
+  test::one "Command failure, no expectation should fail" self::command "" "" 10 && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, --exit=0 should fail" self::command "" "" 10 --exit=0 && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, --exit=10 should succeed" self::command "" "" 10 --exit=10 || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, matching --stderr should succeed" self::command "" "" 10 --exit=10 --stderr= || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, non matching --stderr should fail" self::command "" "" 10 --exit=10 --stderr=lala && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, matching --stdout should succeed" self::command "to stdout" "" 10 --exit=10 --stdout="to stdout" || {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, non matching --stdout should fail" self::command "to stdout" "" 10 --exit=10 --stdout="non matching" && {
+    logger::error "FAIL!"
+    exit 1
+  }
+
+  test::one "Command failure, all expectation match should succeed" self::command "to stdout" "to stderr" 10 --exit=10  --stdout="to stdout" --stderr="to stderr" || {
+    logger::error "FAIL!"
+    exit 1
+  }
+}
+
+>&2 logger::info "Performing self-diagnostic"
+self::test
+>&2 logger::info "All tests successful. Test framework is operational."

examples/tests/test.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+
+# Source the lib
+readonly d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
+# shellcheck source=/dev/null
+. "$d/test-lib.sh"
+
+# Point this to your dagger binary
+readonly DAGGER_BINARY="${DAGGER_BINARY:-$d/../../cmd/dagger/dagger}"
+
+
+test::compute(){
+  local dagger="$1"
+
+  # Compute
+  test::one "Compute: invalid string should fail" --exit=1 --stdout= \
+      "$dagger" compute "$d"/compute/invalid/string
+  test::one "Compute: invalid bool should fail" --exit=1 --stdout= \
+      "$dagger" compute "$d"/compute/invalid/bool
+  test::one "Compute: invalid int should fail" --exit=1 --stdout= \
+      "$dagger" compute "$d"/compute/invalid/int
+  test::one "Compute: invalid struct should fail" --exit=1 --stdout= \
+      "$dagger" compute "$d"/compute/invalid/struct
+  # XXX https://github.com/blocklayerhq/dagger/issues/22
+  #test::one "Compute: noop should succeed" --exit=0 --stdout="{}"  \
+  #    "$dagger" compute "$d"/compute/noop
+  # XXX https://github.com/blocklayerhq/dagger/issues/28
+  #test::one "Compute: unresolved should fail" --exit=1 --stdout=  \
+  #    "$dagger" compute "$d"/compute/invalid/undefined_prop
+  test::one "Compute: simple should succeed" --exit=0 --stdout="{}" \
+      "$dagger" compute "$d"/compute/simple
+}
+
+test::fetchcontainer(){
+  local dagger="$1"
+
+  # Fetch container
+  test::one "FetchContainer: missing ref" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-container/invalid
+  test::one "FetchContainer: non existent container image" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-container/nonexistent/image
+  test::one "FetchContainer: non existent container tag" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-container/nonexistent/tag
+  test::one "FetchContainer: non existent container digest" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-container/nonexistent/digest
+  # XXX https://github.com/blocklayerhq/dagger/issues/32 - this will fail once cached by buildkit
+  # test::one "FetchContainer: non existent container image with valid digest" --exit=1 --stdout= \
+  #    "$dagger" compute "$d"/fetch-container/nonexistent/image-with-valid-digest
+  test::one "FetchContainer: valid containers"       --exit=0 \
+      "$dagger" compute "$d"/fetch-container/exist
+}
+
+test::fetchgit(){
+  local dagger="$1"
+
+  # Fetch git
+  test::one "FetchGit: valid" --exit=0 --stdout="{}" \
+      "$dagger" compute "$d"/fetch-git/exist
+  test::one "FetchGit: invalid" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-git/invalid
+  test::one "FetchGit: non existent remote" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-git/nonexistent/remote
+  test::one "FetchGit: non existent ref" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-git/nonexistent/ref
+  test::one "FetchGit: non existent bork" --exit=1 --stdout= \
+      "$dagger" compute "$d"/fetch-git/nonexistent/bork
+}
+
+test::exec(){
+  # Exec
+  test::one "Exec: invalid" --exit=1 --stdout= \
+      "$dagger" compute "$d"/exec/invalid
+  test::one "Exec: error" --exit=1 --stdout= \
+      "$dagger" compute "$d"/exec/error
+  test::one "Exec: simple" --exit=0 --stdout={} \
+      "$dagger" compute "$d"/exec/simple
+  # XXX should run twice and test that the string "always output" is visible with DOCKER_OUTPUT=1
+  # Alternatively, use export, but this would test multiple things then...
+  test::one "Exec: always" --exit=0 --stdout={} \
+      "$dagger" compute "$d"/exec/always
+  test::one "Exec: env invalid" --exit=1 --stdout= \
+      "$dagger" compute "$d"/exec/env/invalid
+  test::one "Exec: env valid" --exit=0 --stdout={} \
+      "$dagger" compute  "$d"/exec/env/valid
+  # XXX overlays are not wired yet
+  # test::one "Exec: env with overlay" --exit=0 --stdout={} \
+  #    "$dagger" compute --input-string 'bar: "overlay environment"' "$d"/exec/env/overlay
+  # XXX broken right now: https://github.com/blocklayerhq/dagger/issues/30
+  #test::one "Exec: non existent dir" --exit=0 --stdout={} \
+  #    "$dagger" compute  "$d"/exec/dir/doesnotexist
+  #test::one "Exec: valid dir" --exit=0 --stdout={} \
+  #    "$dagger" compute  "$d"/exec/dir/exist
+}
+
+test::all(){
+  local dagger="$1"
+
+  test::compute "$dagger"
+  test::fetchcontainer "$dagger"
+  test::fetchgit "$dagger"
+  test::exec "$dagger"
+
+  # TODO: exec mounts
+  # TODO: copy
+  # TODO: load
+  # TODO: local
+  # TODO: export
+}
+
+case "${1:-all}" in
+  # Help
+  --help)
+    echo "Run all known tests:"
+    echo "  ./test.sh"
+    echo "Run a specific cue module with expectations (all flags are optional if you just expect the command to succeed with no output validation:"
+    echo "  ./test.sh cuefolder --exit=1 --stderr=lala --stdout=foo"
+    ;;
+  # Run all tests
+  "all")
+    test::all "$DAGGER_BINARY"
+    ;;
+  # Anything else means a single / custom test
+  *)
+    test::one "on demand $1" "$DAGGER_BINARY" compute "$@"
+    ;;
+esac