Merge pull request #325 from dagger/bats

tests: use bats for integration tests

.github/workflows/ci.yml

@@ -34,13 +34,9 @@ jobs:
           skip-go-installation: true
           args: --timeout=3m
 
-      - name: Cue Lint
-        run: |
-          make cuelint
-
       - name: Lint
         run: |
-          make check-buildkit-version
+          make shellcheck cuelint check-buildkit-version
 
   test:
     name: Test

.gitignore

@@ -21,3 +21,6 @@
 
 # intellij
 .idea
+
+# node_modules
+tests/node_modules

Makefile

@@ -27,8 +27,12 @@ cuefmt:
 cuelint: cuefmt
 	@test -z "$$(git status -s . | grep -e "^ M"  | grep .cue | cut -d ' ' -f3 | tee /dev/stderr)"
 
+.PHONY: shellcheck
+shellcheck:
+	shellcheck ./tests/*.bats ./tests/*.bash
+
 .PHONY: lint
-lint: cuelint golint check-buildkit-version
+lint: shellcheck cuelint golint check-buildkit-version
 
 .PHONY: check-buildkit-version
 check-buildkit-version:
@@ -41,7 +45,10 @@ check-buildkit-version:
 integration: dagger-debug
 	# Self-diagnostics
 	./tests/test-test.sh 2>/dev/null
-	# Actual integration tests
+	# Bats based integration tests
+	yarn --cwd "./tests" install
+	DAGGER_BINARY="../cmd/dagger/dagger-debug" yarn --cwd "./tests" test
+	# Old style integration tests tests
 	DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all
 
 .PHONY: install

tests/cli.bats

@@ -0,0 +1,144 @@
+setup() {
+    load 'helpers'
+
+    common_setup
+}
+
+@test "dagger list" {
+    run "$DAGGER" list
+    assert_success
+    assert_output ""
+
+    "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple
+
+    run "$DAGGER" list
+    assert_success
+    assert_output --partial "simple"
+}
+
+@test "dagger new --plan-dir" {
+    run "$DAGGER" list
+    assert_success
+    assert_output ""
+
+    "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple
+
+    # duplicate name
+    run "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple
+    assert_failure
+
+    # verify the plan works
+    "$DAGGER" up -d "simple"
+
+    # verify we have the right plan
+    run "$DAGGER" query -f cue -d "simple" -c -f json
+    assert_success
+    assert_output --partial '{
+  "bar": "another value",
+  "computed": "test",
+  "foo": "value"
+}'
+}
+
+@test "dagger new --plan-git" {
+    "$DAGGER" new --plan-git https://github.com/samalba/dagger-test.git simple
+    "$DAGGER" up -d "simple"
+    run "$DAGGER" query -f cue -d "simple" -c
+    assert_success
+    assert_output --partial '{
+    foo: "value"
+    bar: "another value"
+}'
+}
+
+@test "dagger query" {
+    "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple
+    run "$DAGGER" query -l error -d "simple"
+    assert_success
+    assert_output '{
+  "bar": "another value",
+  "foo": "value"
+}'
+    # concrete should fail at this point since we haven't up'd
+    run "$DAGGER" query -d "simple" -c
+    assert_failure
+
+    # target
+    run "$DAGGER" -l error query -d "simple" foo
+    assert_success
+    assert_output '"value"'
+
+    # ensure computed values show up
+    "$DAGGER" up -d "simple"
+    run "$DAGGER" -l error query -d "simple"
+    assert_success
+    assert_output --partial '"computed": "test"'
+
+    # concrete should now work
+    "$DAGGER" query -d "simple" -c
+
+    # --no-computed should yield the same result as before
+    run "$DAGGER" query -l error --no-computed -d "simple"
+    assert_success
+    assert_output '{
+  "bar": "another value",
+  "foo": "value"
+}'
+
+    # --no-plan should give us only the computed values
+    run "$DAGGER" query -l error --no-plan -d "simple"
+    assert_success
+    assert_output '{
+  "computed": "test"
+}'
+}
+
+@test "dagger plan" {
+    "$DAGGER" new --plan-dir "$TESTDIR"/cli/simple simple
+
+    # plan dir
+    "$DAGGER" -d "simple" plan dir "$TESTDIR"/cli/simple
+    run "$DAGGER" -d "simple" query
+    assert_success
+    assert_output --partial '"foo": "value"'
+
+    # plan git
+    "$DAGGER" -d "simple" plan git https://github.com/samalba/dagger-test.git
+    run "$DAGGER" -d "simple" query
+    assert_success
+    assert_output --partial '"foo": "value"'
+}
+
+@test "dagger input" {
+    "$DAGGER" new --plan-dir "$TESTDIR"/cli/input "input"
+
+    # missing input
+    "$DAGGER"  up -d "input"
+    run "$DAGGER" -l error query -d "input"
+    assert_success
+    assert_output '{
+  "foo": "bar"
+}'
+
+    # input dir
+    "$DAGGER" input -d "input" dir "source" "$TESTDIR"/cli/input/testdata
+    "$DAGGER" "${DAGGER_BINARY_ARGS[@]}" up -d "input"
+    "$DAGGER"  up -d "input"
+    run "$DAGGER" -l error query -d "input"
+    assert_success
+    assert_output '{
+  "bar": "thisisatest\n",
+  "foo": "bar",
+  "source": {}
+}'
+
+    # input git
+    "$DAGGER" input -d "input" git "source" https://github.com/samalba/dagger-test-simple.git
+    "$DAGGER" up -d "input"
+    run "$DAGGER" -l error query -d "input"
+    assert_output '{
+  "bar": "testgit\n",
+  "foo": "bar",
+  "source": {}
+}'
+}

tests/helpers.bash

@@ -0,0 +1,12 @@
+common_setup() {
+    load 'node_modules/bats-support/load'
+    load 'node_modules/bats-assert/load'
+
+    TESTDIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )"
+
+    DAGGER="${DAGGER_BINARY:-$TESTDIR/../cmd/dagger/dagger}"
+    export DAGGER
+
+    DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
+    export DAGGER_STORE
+}

tests/package.json

@@ -0,0 +1,11 @@
+{
+  "license": "Apache-2.0",
+  "scripts": {
+    "test": "bats ."
+  },
+  "devDependencies": {
+    "bats": "https://github.com/bats-core/bats-core.git",
+    "bats-assert": "https://github.com/bats-core/bats-assert",
+    "bats-support": "https://github.com/bats-core/bats-support"
+  }
+}

tests/test-cli.sh

@@ -1,208 +0,0 @@
-#!/usr/bin/env bash
-set -o errexit -o errtrace -o functrace -o nounset -o pipefail
-
-# Test Directory
-d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
-
-test::cli() {
-  local dagger="$1"
-
-  test::cli::list "$dagger"
-  test::cli::newdir "$dagger"
-  test::cli::newgit "$dagger"
-  test::cli::query "$dagger"
-  test::cli::plan "$dagger"
-  test::cli::input "$dagger"
-}
-
-test::cli::list() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: list: no deployments" --stdout="" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" list
-
-  test::one "CLI: list: create deployment" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/simple simple
-
-  test::one "CLI: list: with deployments" --stdout="simple" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" list
-}
-
-test::cli::newdir() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: new: --plan-dir" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/simple simple
-
-  test::one "CLI: new: duplicate name" --exit=1 \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/simple simple
-
-  test::one "CLI: new: verify plan can be upped" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "simple"
-
-  test::one "CLI: new: verify we have the right plan" --stdout='{
-  "bar": "another value",
-  "computed": "test",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -f cue -d "simple" -c -f json
-}
-
-test::cli::newgit() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: new git: --plan-git" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-git https://github.com/samalba/dagger-test.git simple
-
-  test::one "CLI: new git: verify plan can be upped" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "simple"
-
-  test::one "CLI: new git: verify we have the right plan" --stdout='{
-    foo: "value"
-    bar: "another value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -f cue -d "simple" -c
-}
-
-test::cli::query() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: query: initialize simple" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/simple simple
-
-  test::one "CLI: query: before up" --stdout='{
-  "bar": "another value",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple"
-
-  test::one "CLI: query: concrete should fail" --exit=1 \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" -c
-
-  test::one "CLI: query: target" --stdout='"value"' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" foo
-
-  test::one "CLI: query: compute missing values" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "simple"
-
-  test::one "CLI: query: all values" --stdout='{
-  "bar": "another value",
-  "computed": "test",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple"
-
-  test::one "CLI: query: concrete should work" --stdout='{
-  "bar": "another value",
-  "computed": "test",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" -c
-
-  test::one "CLI: query --no-computed" --stdout='{
-  "bar": "another value",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" --no-computed
-
-  test::one "CLI: query: --no-plan" --stdout='{
-  "computed": "test"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" -c --no-plan
-}
-
-test::cli::plan() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: new" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/simple simple
-
-  test::one "CLI: plan dir" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" -d "simple" plan dir "$d"/cli/simple
-
-  test::one "CLI: plan dir: verify we have the right plan" --stdout='{
-  "bar": "another value",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" -d "simple" query
-
-  test::one "CLI: plan git" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" -d "simple" plan git https://github.com/samalba/dagger-test.git
-
-  test::one "CLI: plan git: verify we have the right plan" --stdout='{
-  "bar": "another value",
-  "foo": "value"
-}' \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "simple" -c
-}
-
-test::cli::input() {
-  local dagger="$1"
-
-  # Create temporary store
-  local DAGGER_STORE
-  DAGGER_STORE="$(mktemp -d -t dagger-store-XXXXXX)"
-  export DAGGER_STORE
-
-  test::one "CLI: input: new" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" new --plan-dir "$d"/cli/input "input"
-
-  test::one "CLI: input: up missing input" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "input"
-
-  test::one "CLI: input: query missing input" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "input" --stdout='{
-  "foo": "bar"
-}'
-
-  test::one "CLI: input: set dir" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" input -d "input" dir "source" ./tests/cli/input/testdata
-
-  test::one "CLI: input: up with input dir" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "input"
-
-  test::one "CLI: input: query with input dir" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "input" --stdout='{
-  "bar": "thisisatest\n",
-  "foo": "bar",
-  "source": {}
-}'
-
-  test::one "CLI: input: set git" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" input -d "input" git "source" https://github.com/samalba/dagger-test-simple.git
-
-  test::one "CLI: input: up with input git" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" up -d "input"
-
-  test::one "CLI: query with input git" \
-      "$dagger" "${DAGGER_BINARY_ARGS[@]}" query -d "input" --stdout='{
-  "bar": "testgit\n",
-  "foo": "bar",
-  "source": {}
-}'
-}

tests/test.sh

@@ -22,8 +22,6 @@ d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
 # shellcheck source=/dev/null
 . "$d/test-stdlib.sh"
 # shellcheck source=/dev/null
-. "$d/test-cli.sh"
-# shellcheck source=/dev/null
 . "$d/test-examples.sh"
 
 test::all(){
@@ -32,7 +30,6 @@ test::all(){
   test::suite "compute" && test::compute "$dagger"
   test::suite "llb" && test::llb "$dagger"
   test::suite "stdlib" && test::stdlib "$dagger"
-  test::suite "cli" && test::cli "$dagger"
   test::suite "examples" && test::examples "$dagger"
 }
 

tests/yarn.lock

@@ -0,0 +1,15 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"bats-assert@https://github.com/bats-core/bats-assert":
+  version "2.0.0"
+  resolved "https://github.com/bats-core/bats-assert#0a8dd57e2cc6d4cc064b1ed6b4e79b9f7fee096f"
+
+"bats-support@https://github.com/bats-core/bats-support":
+  version "0.3.0"
+  resolved "https://github.com/bats-core/bats-support#d140a65044b2d6810381935ae7f0c94c7023c8c3"
+
+"bats@https://github.com/bats-core/bats-core.git":
+  version "1.3.0"
+  resolved "https://github.com/bats-core/bats-core.git#b81de286448ae3a11e3342783dcdc1b08765a421"