Merge pull request #335 from dagger/stdlib-bats

tests: migrate stdlib tests to bats
This commit is contained in:
Andrea Luzzardi 2021-04-15 16:14:59 -07:00 committed by GitHub
commit 8f844366d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 51 additions and 350 deletions

View File

@ -43,13 +43,8 @@ check-buildkit-version:
.PHONY: integration .PHONY: integration
integration: dagger-debug integration: dagger-debug
# Self-diagnostics
./tests/test-test.sh 2>/dev/null
# Bats based integration tests
yarn --cwd "./tests" install yarn --cwd "./tests" install
DAGGER_BINARY="../cmd/dagger/dagger-debug" yarn --cwd "./tests" test DAGGER_BINARY="../cmd/dagger/dagger-debug" yarn --cwd "./tests" test
# Old style integration tests tests
DAGGER_BINARY="./cmd/dagger/dagger-debug" time ./tests/test.sh all
.PHONY: install .PHONY: install
install: dagger install: dagger

View File

@ -15,6 +15,14 @@ common_setup() {
} }
skip_unless_secrets_available() { skip_unless_secrets_available() {
local inputFile="$1" local inputFile="$1"
sops exec-file "$inputFile" echo > /dev/null 2>&1 || skip "$inputFile cannot be decrypted" sops exec-file "$inputFile" echo > /dev/null 2>&1 || skip "$inputFile cannot be decrypted"
} }
skip_unless_local_kube() {
if [ -f ~/.kube/config ] && grep -q "kind" ~/.kube/config &> /dev/null; then
echo "Kubernetes available"
else
skip "local kubernetes cluster not available"
fi
}

41
tests/stdlib.bats Normal file
View File

@ -0,0 +1,41 @@
setup() {
load 'helpers'
common_setup
}
@test "stdlib: alpine" {
"$DAGGER" compute "$TESTDIR"/stdlib/alpine
}
@test "stdlib: react" {
"$DAGGER" compute "$TESTDIR"/stdlib/js/react --input-dir TestData="$TESTDIR"/stdlib/js/react/testdata
}
@test "stdlib: go" {
"$DAGGER" compute "$TESTDIR"/stdlib/go --input-dir TestData="$TESTDIR"/stdlib/go/testdata
}
@test "stdlib: file" {
"$DAGGER" compute "$TESTDIR"/stdlib/file
}
@test "stdlib: netlify" {
skip_unless_secrets_available "$TESTDIR"/stdlib/netlify/inputs.yaml
"$DAGGER" compute "$TESTDIR"/stdlib/netlify --input-yaml "$TESTDIR"/stdlib/netlify/inputs.yaml
}
@test "stdlib: kubernetes" {
skip "disabled"
skip_unless_local_kube
"$DAGGER" compute "$TESTDIR"/stdlib/kubernetes --input-dir kubeconfig=~/.kube
}
@test "stdlib: helm" {
skip "disabled"
skip_unless_local_kube
"$DAGGER" compute "$TESTDIR"/stdlib/kubernetes/helm --input-dir kubeconfig=~/.kube --input-dir TestHelmSimpleChart.deploy.chartSource="$TESTDIR"/stdlib/kubernetes/helm/testdata/mychart
}

View File

@ -1,145 +0,0 @@
#!/usr/bin/env bash
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
########################################################################
# Logging helpers
########################################################################
readonly COLOR_RED=1
readonly COLOR_GREEN=2
readonly COLOR_YELLOW=3
# Prefix a date to a log line and output to stderr
logger::stamp(){
local color="$1"
local level="$2"
local i
shift
shift
[ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput setaf "$color"
for i in "$@"; do
>&2 printf "[%s] [%s] %s\\n" "$(date)" "$level" "$i"
done
[ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op
}
logger::info(){
logger::stamp "$COLOR_GREEN" "INFO" "$@"
}
logger::warning(){
logger::stamp "$COLOR_YELLOW" "WARNING" "$@"
}
logger::error(){
logger::stamp "$COLOR_RED" "ERROR" "$@"
}
########################################################################
# Handle exit code and errors of random command
########################################################################
wrap::err(){
local args=()
local actualExit=0
local expectedExit=0
local ret=0
for i in "$@"; do
if [ "${i:0:9}" == "--stderr=" ]; then
local expectedStderr="${i:9}"
elif [ "${i:0:7}" == "--exit=" ]; then
expectedExit="${i:7}"
expectedExit="${expectedExit:-0}"
else
args+=("$i")
fi
done
logger::info " -> ${args[*]}"
exec 3>&1
actualStderr="$("${args[@]}" 2>&1 1>&3)" || actualExit="$?"
[ "$expectedExit" == "$actualExit" ] || {
logger::error " -> Expected exit code: $expectedExit" " -> Actual exit code : $actualExit"
logger::error " -> Stderr was:"
>&2 jq <<<"$actualStderr" 2>/dev/null || {
>&2 echo "$actualStderr"
logger::error " -> Also, stderr is not json"
}
ret=1
}
[ -z "${expectedStderr+x}" ] || [ "$expectedStderr" == "$actualStderr" ] || {
logger::error " -> Expected stderr:"
>&2 jq <<<"$expectedStderr" 2>/dev/null || {
>&2 echo "$expectedStderr"
}
logger::error " -> Actual stderr :"
>&2 jq <<<"$actualStderr" 2>/dev/null || {
>&2 echo "$actualStderr"
logger::error " -> Also, stderr is not json ^"
}
ret=1
}
exec 3>&-
return "$ret"
}
########################################################################
# Main test function
# argument 1 is a test description
# to test the exit code, pass --exit=int (if not provided, will test that the command exits succesfully)
# to test the value of stderr, pass --stderr=string (if not provided, stderr is not verified)
# to test the value of stdout, pass --stdout=string (if not provided, stdout is not verified)
# any other argument is the command that is going to be run
# Example:
# test dagger compute somecue --exit=1 --stderr=expectederror
########################################################################
test::one(){
local testDescription="$1"
shift
local args=()
local ret=0
for i in "$@"; do
if [ "${i:0:9}" == "--stdout=" ]; then
local expectedStdout="${i:9}"
else
args+=("$i")
fi
done
logger::info "$testDescription"
local actualStdout
actualStdout="$(wrap::err "${args[@]}")" || {
ret=1
}
[ -z "${expectedStdout+x}" ] || [ "$expectedStdout" == "$actualStdout" ] || {
exec 3>&-
logger::error " -> Expected stdout: $expectedStdout" " -> Actual stdout : $actualStdout"
ret=1
}
[ "$ret" != 0 ] || logger::info " -> Success"
return "$ret"
}
# Similar to test::one, however tests will be skipped if secrets cannot be decrypted
test::secret(){
local inputFile="$1"
shift
# if sops exec-file "$inputFile" echo > /dev/null 2>&1; then
if sops exec-file "$inputFile" echo; then
test::one "$@" --input-yaml "$inputFile"
else
logger::warning "Skip \"$1\": secrets not available"
fi
}
disable(){
logger::warning "Test \"$2\" has been disabled."
}

View File

@ -1,31 +0,0 @@
#!/usr/bin/env bash
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
# Test Directory
d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
test::stdlib(){
local dagger="$1"
test::one "stdlib: alpine" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/alpine
test::one "stdlib: react" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/js/react --input-dir TestData="$d"/stdlib/js/react/testdata
test::one "stdlib: go" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/go --input-dir TestData="$d"/stdlib/go/testdata
test::one "stdlib: file" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/file
test::secret "$d"/stdlib/netlify/inputs.yaml "stdlib: netlify" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/netlify
# Check if there is a kubeconfig and if it for a kind cluster
if [ -f ~/.kube/config ] && grep -q "kind" ~/.kube/config &> /dev/null; then
test::one "stdlib: kubernetes" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/kubernetes --input-dir kubeconfig=~/.kube
test::one "stdlib: helm" \
"$dagger" "${DAGGER_BINARY_ARGS[@]}" compute "$d"/stdlib/kubernetes/helm --input-dir kubeconfig=~/.kube --input-dir TestHelmSimpleChart.deploy.chartSource="$d"/stdlib/kubernetes/helm/testdata/mychart
else
logger::warning "Skip kubernetes test: local cluster not available"
fi
}

View File

@ -1,112 +0,0 @@
#!/usr/bin/env bash
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
# The purpose of this is solely to test::one the test::one framework in test-lib.sh
readonly d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
# Performing self-diagnostic linting check first
shellcheck "$d/"*.sh
# shellcheck source=/dev/null
. "$d/test-lib.sh"
########################################################################
# Verifying the test::one framework is working
########################################################################
self::command(){
local stdout="${1:-}"
local stderr="${2:-}"
local ret="${3:-0}"
printf "%s" "$stdout"
>&2 printf "%s" "$stderr"
exit "$ret"
}
self::test(){
# Command success testing
test::one "Command success, no expectation should succeed" self::command || {
logger::error "FAIL!"
exit 1
}
test::one "Command success, --exit=0 should succeed" self::command --exit=0 || {
logger::error "FAIL!"
exit 1
}
test::one "Command success, --exit=1 should fail" self::command --exit=1 && {
logger::error "FAIL!"
exit 1
}
test::one "Command success, matching --stderr should succeed" self::command "" "to err" --stderr="to err" || {
logger::error "FAIL!"
exit 1
}
test::one "Command success, non matching --stderr should fail" self::command --stderr="to stderr" && {
logger::error "FAIL!"
exit 1
}
test::one "Command success, matching --stdout foo should succeed" self::command "lol foo" --stdout="lol foo" || {
logger::error "FAIL!"
exit 1
}
test::one "Command success, non matching --stdout should fail" self::command "lol foo" --stdout="lol" && {
logger::error "FAIL!"
exit 1
}
test::one "Command success, all expectation match should succeed" self::command "lol" --exit=0 --stdout="lol" --stderr= || {
logger::error "FAIL!"
exit 1
}
# Command failure testing
test::one "Command failure, no expectation should fail" self::command "" "" 10 && {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, --exit=0 should fail" self::command "" "" 10 --exit=0 && {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, --exit=10 should succeed" self::command "" "" 10 --exit=10 || {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, matching --stderr should succeed" self::command "" "" 10 --exit=10 --stderr= || {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, non matching --stderr should fail" self::command "" "" 10 --exit=10 --stderr=lala && {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, matching --stdout should succeed" self::command "to stdout" "" 10 --exit=10 --stdout="to stdout" || {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, non matching --stdout should fail" self::command "to stdout" "" 10 --exit=10 --stdout="non matching" && {
logger::error "FAIL!"
exit 1
}
test::one "Command failure, all expectation match should succeed" self::command "to stdout" "to stderr" 10 --exit=10 --stdout="to stdout" --stderr="to stderr" || {
logger::error "FAIL!"
exit 1
}
}
>&2 logger::info "Performing self-diagnostic"
self::test
>&2 logger::info "All tests successful. Test framework is operational."

View File

@ -1,55 +0,0 @@
#!/usr/bin/env bash
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
readonly SUITE="${SUITE:-"all"}"
# Point this to your dagger binary
readonly DAGGER_BINARY="${DAGGER_BINARY:-$d/../cmd/dagger/dagger}"
# The default arguments are a no-op, but having "anything" is a little cheat necessary for "${DAGGER_BINARY_ARGS[@]}" to not be empty down there
DAGGER_BINARY_ARGS="${DAGGER_BINARY_ARGS:---log-format json}"
read -ra DAGGER_BINARY_ARGS <<< "${DAGGER_BINARY_ARGS:-}"
readonly DAGGER_BINARY_ARGS
# Test Directory
d=$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)
# Source the lib
# shellcheck source=/dev/null
. "$d/test-lib.sh"
# shellcheck source=/dev/null
. "$d/test-stdlib.sh"
test::all(){
local dagger="$1"
test::suite "stdlib" && test::stdlib "$dagger"
}
test::suite() {
local name="$1"
{
# if SUITE is "-foo", run everything that's not "foo"
[[ "$SUITE" = -* ]] && [[ "$SUITE" != "-${name}" ]]
} || {
# Run a specific suite, or match all of them if suite is "all"
[ "${SUITE}" = "all" ] || [ "${SUITE}" = "${name}" ]
} || false
}
case "${1:-all}" in
# Help
--help)
echo "Run all known tests:"
echo " ./test.sh"
echo "Run a specific cue module with expectations (all flags are optional if you just expect the command to succeed with no output validation:"
echo " ./test.sh cuefolder --exit=1 --stderr=lala --stdout=foo"
;;
# Run all tests
"all")
test::all "$DAGGER_BINARY"
;;
# Anything else means a single / custom test
*)
test::one "on demand $1" "$DAGGER_BINARY" compute "$@"
;;
esac