From 6b7c9787caf509fe91fd02f033f5d3390092a1d8 Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Tue, 26 Sep 2017 19:33:45 -0700 Subject: [PATCH 1/3] Added the todo command for marking tests that may fail as passing, potentially (in pretty print) as a "bonus" if they pass. --- libexec/bats | 2 +- libexec/bats-exec-test | 25 ++++++++++----- libexec/bats-format-tap-stream | 20 ++++++++++-- man/bats.7.ronn | 35 +++++++++++++++++++++ test/bats.bats | 26 +++++++++++++++ test/fixtures/bats/todo_empty_failing.bats | 4 +++ test/fixtures/bats/todo_empty_passing.bats | 4 +++ test/fixtures/bats/todo_reason_failing.bats | 4 +++ test/fixtures/bats/todo_reason_passing.bats | 4 +++ 9 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/bats/todo_empty_failing.bats create mode 100644 test/fixtures/bats/todo_empty_passing.bats create mode 100644 test/fixtures/bats/todo_reason_failing.bats create mode 100644 test/fixtures/bats/todo_reason_passing.bats diff --git a/libexec/bats b/libexec/bats index 71f392f7..e2104dbe 100755 --- a/libexec/bats +++ b/libexec/bats @@ -2,7 +2,7 @@ set -e version() { - echo "Bats 0.4.0" + echo "Bats 0.5.0" } usage() { diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 8f3bd510..b60c4320 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -79,6 +79,10 @@ skip() { exit 0 } +todo() { + BATS_TEST_TODO=${1:-1} +} + bats_test_begin() { BATS_TEST_DESCRIPTION="$1" if [ -n "$BATS_EXTENDED_SYNTAX" ]; then @@ -240,27 +244,34 @@ bats_teardown_trap() { } bats_exit_trap() { - local status - local skipped + local status=0 + local skipped="" + local todo="" + local fail_status=1 trap - err exit - skipped="" if [ -n "$BATS_TEST_SKIPPED" ]; then + fail_status=0 skipped=" # skip" if [ "1" != "$BATS_TEST_SKIPPED" ]; then skipped+=" ($BATS_TEST_SKIPPED)" fi + elif [ -n "$BATS_TEST_TODO" ]; then + fail_status=0 + todo=" # todo" + if [ "1" != "$BATS_TEST_TODO" ]; then + todo+=" $BATS_TEST_TODO" + fi fi if [ -z "$BATS_TEST_COMPLETED" ] || [ -z "$BATS_TEARDOWN_COMPLETED" ]; then - echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + echo "not ok ${BATS_TEST_NUMBER} ${BATS_TEST_DESCRIPTION}${todo}" >&3 bats_print_stack_trace "${BATS_ERROR_STACK_TRACE[@]}" >&3 bats_print_failed_command "${BATS_ERROR_STACK_TRACE[${#BATS_ERROR_STACK_TRACE[@]}-1]}" "$BATS_ERROR_STATUS" >&3 sed -e "s/^/# /" < "$BATS_OUT" >&3 - status=1 + status="$fail_status" else - echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 - status=0 + echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}${todo}" >&3 fi rm -f "$BATS_OUT" diff --git a/libexec/bats-format-tap-stream b/libexec/bats-format-tap-stream index 614768f4..977970a7 100755 --- a/libexec/bats-format-tap-stream +++ b/libexec/bats-format-tap-stream @@ -64,6 +64,14 @@ log() { clear_color } +todone() { + local reason="$1" + [ -z "$reason" ] || reason=": $reason" + go_to_column 0 + printf " ✓ %s ✭ DONE%s" "$name" "$reason" + advance +} + summary() { printf "\n%d test%s" "$count" "$(plural "$count")" @@ -145,16 +153,24 @@ while IFS= read -r line; do ;; "ok "* ) skip_expr="ok $index # skip (\(([^)]*)\))?" + todo_expr="ok $index .*# todo([[:space:]]+(.+))?" if [[ "$line" =~ $skip_expr ]]; then let skipped+=1 buffer skip "${BASH_REMATCH[2]}" + elif [[ "$line" =~ $todo_expr ]]; then + buffer todone "${BASH_REMATCH[2]}" else buffer pass fi ;; "not ok "* ) - let failures+=1 - buffer fail + todo_expr="ok $index .*# todo([[:space:]]+(.+))?" + if [[ "$line" =~ $todo_expr ]]; then + buffer pass + else + let failures+=1 + buffer fail + fi ;; "# "* ) buffer log "${line:2}" diff --git a/man/bats.7.ronn b/man/bats.7.ronn index 7f6dd184..3ec76a3e 100644 --- a/man/bats.7.ronn +++ b/man/bats.7.ronn @@ -110,6 +110,41 @@ Or you can skip conditionally: } +THE TODO COMMAND +---------------- + +Tests can be marked as TODO by using the `todo` command at the point in a test before it completes. + +A passing test marked with TODO should be considered a "bonus", but both pass and fail are considered successful. + +See [the TAP specification](http://testanything.org/tap-specification.html) for more details on this feature. + + @test "A test that may fail" { + todo + run foo + [ "$status" -eq 0 ] + } + +Optionally, you may include a reason for marking TODO: + + @test "A test that may fail" { + todo "This command will return zero soon, but not now" + run foo + [ "$status" -eq 0 ] + } + +Or you can mark todo conditionally: + + @test "A test which usually fails on a mac" { + if [ $(uname -s) == Darwin ]; then + todo "Get this working on a Mac" + fi + + run foo + [ "$status" -eq 0 ] + } + + SETUP AND TEARDOWN FUNCTIONS ---------------------------- diff --git a/test/bats.bats b/test/bats.bats index f1aff293..084f59fc 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -262,3 +262,29 @@ fixtures bats [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 loop_func" ] } + +@test "empty passing todo" { + run bats -p "$FIXTURE_ROOT/todo_empty_passing.bats" + [ $status -eq 0 ] + [[ "${lines[0]}" =~ "✭ DONE" ]] + [ "${lines[2]}" = "1 test, 0 failures" ] +} + +@test "empty failing todo" { + run bats -p "$FIXTURE_ROOT/todo_empty_failing.bats" + [ $status -eq 0 ] + [ "${lines[4]}" = "1 test, 0 failures" ] +} + +@test "passing todo with reason" { + run bats -p "$FIXTURE_ROOT/todo_reason_passing.bats" + [ $status -eq 0 ] + [[ "${lines[0]}" =~ "✭ DONE" ]] + [ "${lines[2]}" = "1 test, 0 failures" ] +} + +@test "failing todo with reason" { + run bats -p "$FIXTURE_ROOT/todo_reason_failing.bats" + [ $status -eq 0 ] + [ "${lines[4]}" = "1 test, 0 failures" ] +} diff --git a/test/fixtures/bats/todo_empty_failing.bats b/test/fixtures/bats/todo_empty_failing.bats new file mode 100644 index 00000000..a2116034 --- /dev/null +++ b/test/fixtures/bats/todo_empty_failing.bats @@ -0,0 +1,4 @@ +@test "Flaky failing test no reason" { + todo + false +} \ No newline at end of file diff --git a/test/fixtures/bats/todo_empty_passing.bats b/test/fixtures/bats/todo_empty_passing.bats new file mode 100644 index 00000000..3dc9f41f --- /dev/null +++ b/test/fixtures/bats/todo_empty_passing.bats @@ -0,0 +1,4 @@ +@test "Flaky passing test no reason" { + todo + true +} \ No newline at end of file diff --git a/test/fixtures/bats/todo_reason_failing.bats b/test/fixtures/bats/todo_reason_failing.bats new file mode 100644 index 00000000..d2497ccc --- /dev/null +++ b/test/fixtures/bats/todo_reason_failing.bats @@ -0,0 +1,4 @@ +@test "Flaky failing test with reason" { + todo "some really good reason" + false +} \ No newline at end of file diff --git a/test/fixtures/bats/todo_reason_passing.bats b/test/fixtures/bats/todo_reason_passing.bats new file mode 100644 index 00000000..6c4d72f4 --- /dev/null +++ b/test/fixtures/bats/todo_reason_passing.bats @@ -0,0 +1,4 @@ +@test "Flaky passing test with reason" { + todo "some really good reason" + true +} \ No newline at end of file From 21277a33ad343f16d14cb45e543dbb22306e75cd Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Tue, 26 Sep 2017 19:43:27 -0700 Subject: [PATCH 2/3] Added changelog. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 235bf1ee..ef447875 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,10 @@ on the wiki. ## Version history +*0.5.0* (September, 2017) + +* Added todo directive as an alternative to skip to let tests run, but pass regardless of their status. + *0.4.0* (August 13, 2014) * Improved the display of failing test cases. Bats now shows the From a612c1185c1ba591aef6852db40640c03ebeb2f0 Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Wed, 27 Sep 2017 07:50:07 -0700 Subject: [PATCH 3/3] Updated re-ran ronn to generate man page. --- man/bats.7 | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/man/bats.7 b/man/bats.7 index d0836e52..53e39120 100644 --- a/man/bats.7 +++ b/man/bats.7 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BATS" "7" "November 2013" "" "" +.TH "BATS" "7" "September 2017" "" "" . .SH "NAME" \fBbats\fR \- Bats test file format @@ -142,6 +142,66 @@ Or you can skip conditionally: . .IP "" 0 . +.SH "THE TODO COMMAND" +Tests can be marked as TODO by using the \fBtodo\fR command at the point in a test before it completes\. +. +.P +A passing test marked with TODO should be considered a "bonus", but both pass and fail are considered successful\. +. +.P +See the TAP specification \fIhttp://testanything\.org/tap\-specification\.html\fR for more details on this feature\. +. +.IP "" 4 +. +.nf + +@test "A test that may fail" { + todo + run foo + [ "$status" \-eq 0 ] +} +. +.fi +. +.IP "" 0 +. +.P +Optionally, you may include a reason for marking TODO: +. +.IP "" 4 +. +.nf + +@test "A test that may fail" { + todo "This command will return zero soon, but not now" + run foo + [ "$status" \-eq 0 ] +} +. +.fi +. +.IP "" 0 +. +.P +Or you can mark todo conditionally: +. +.IP "" 4 +. +.nf + +@test "A test which usually fails on a mac" { + if [ $(uname \-s) == Darwin ]; then + todo "Get this working on a Mac" + fi + + run foo + [ "$status" \-eq 0 ] +} +. +.fi +. +.IP "" 0 +. .SH "SETUP AND TEARDOWN FUNCTIONS" You can define special \fBsetup\fR and \fBteardown\fR functions which run before and after each test case, respectively\. Use these to load fixtures, set up your environment, and clean up when you\'re done\. .