Sane use of test_expect_failure

Originally, test_expect_failure was designed to be the opposite
of test_expect_success, but this was a bad decision.  Most tests
run a series of commands that leads to the single command that
needs to be tested, like this:

    test_expect_{success,failure} 'test title' '
	setup1 &&
        setup2 &&
        setup3 &&
        what is to be tested
    '

And expecting a failure exit from the whole sequence misses the
point of writing tests.  Your setup$N that are supposed to
succeed may have failed without even reaching what you are
trying to test.  The only valid use of test_expect_failure is to
check a trivial single command that is expected to fail, which
is a minority in tests of Porcelain-ish commands.

This large-ish patch rewrites all uses of test_expect_failure to
use test_expect_success and rewrites the condition of what is
tested, like this:

    test_expect_success 'test title' '
	setup1 &&
        setup2 &&
        setup3 &&
        ! this command should fail
    '

test_expect_failure is redefined to serve as a reminder that
that test *should* succeed but due to a known breakage in git it
currently does not pass.  So if git-foo command should create a
file 'bar' but you discovered a bug that it doesn't, you can
write a test like this:

    test_expect_failure 'git-foo should create bar' '
        rm -f bar &&
        git foo &&
        test -f bar
    '

This construct acts similar to test_expect_success, but instead
of reporting "ok/FAIL" like test_expect_success does, the
outcome is reported as "FIXED/still broken".

Signed-off-by: Junio C Hamano <gitster@pobox.com>

t/README

@@ -160,14 +160,12 @@ library for your script to use.
 
  - test_expect_failure <message> <script>
 
-   This is the opposite of test_expect_success.  If <script>
-   yields success, test is considered a failure.
-
-   Example:
-
-	test_expect_failure \
-	    'git-update-index without --add should fail adding.' \
-	    'git-update-index should-be-empty'
+   This is NOT the opposite of test_expect_success, but is used
+   to mark a test that demonstrates a known breakage.  Unlike
+   the usual test_expect_success tests, which say "ok" on
+   success and "FAIL" on failure, this will say "FIXED" on
+   success and "still broken" on failure.  Failures from these
+   tests won't cause -i (immediate) to stop.
 
  - test_debug <script>
 

t/t0000-basic.sh

@@ -46,13 +46,25 @@ test_expect_success \
     '.git/objects should have 3 subdirectories.' \
     'test $(wc -l < full-of-directories) = 3'
 
+################################################################
+# Test harness
+test_expect_success 'success is reported like this' '
+    :
+'
+test_expect_failure 'pretend we have a known breakage' '
+    false
+'
+test_expect_failure 'pretend we have fixed a known breakage' '
+    :
+'
+
 ################################################################
 # Basics of the basics
 
 # updating a new file without --add should fail.
-test_expect_failure \
-    'git update-index without --add should fail adding.' \
-    'git update-index should-be-empty'
+test_expect_success 'git update-index without --add should fail adding.' '
+    ! git update-index should-be-empty
+'
 
 # and with --add it should succeed, even if it is empty (it used to fail).
 test_expect_success \
@@ -70,9 +82,9 @@ test_expect_success \
 
 # Removing paths.
 rm -f should-be-empty full-of-directories
-test_expect_failure \
-    'git update-index without --remove should fail removing.' \
-    'git update-index should-be-empty'
+test_expect_success 'git update-index without --remove should fail removing.' '
+    ! git update-index should-be-empty
+'
 
 test_expect_success \
     'git update-index with --remove should be able to remove.' \
@@ -204,9 +216,9 @@ test_expect_success \
     'put invalid objects into the index.' \
     'git update-index --index-info < badobjects'
 
-test_expect_failure \
-    'writing this tree without --missing-ok.' \
-    'git write-tree'
+test_expect_success 'writing this tree without --missing-ok.' '
+    ! git write-tree
+'
 
 test_expect_success \
     'writing this tree with --missing-ok.' \

t/t0030-stripspace.sh

@@ -243,14 +243,14 @@ test_expect_success \
     test `printf "$ttt$sss$sss$sss" | git stripspace | wc -l` -gt 0
 '
 
-test_expect_failure \
+test_expect_success \
     'text plus spaces without newline at end should not show spaces' '
-    printf "$ttt$sss" | git stripspace | grep -q "  " ||
-    printf "$ttt$ttt$sss" | git stripspace | grep -q "  " ||
-    printf "$ttt$ttt$ttt$sss" | git stripspace | grep -q "  " ||
-    printf "$ttt$sss$sss" | git stripspace | grep -q "  " ||
-    printf "$ttt$ttt$sss$sss" | git stripspace | grep -q "  " ||
-    printf "$ttt$sss$sss$sss" | git stripspace | grep -q "  "
+    ! (printf "$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (printf "$ttt$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (printf "$ttt$ttt$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (printf "$ttt$sss$sss" | git stripspace | grep -q "  ") &&
+    ! (printf "$ttt$ttt$sss$sss" | git stripspace | grep -q "  ") &&
+    ! (printf "$ttt$sss$sss$sss" | git stripspace | grep -q "  ")
 '
 
 test_expect_success \
@@ -280,14 +280,14 @@ test_expect_success \
     git diff expect actual
 '
 
-test_expect_failure \
+test_expect_success \
     'text plus spaces at end should not show spaces' '
-    echo "$ttt$sss" | git stripspace | grep -q "  " ||
-    echo "$ttt$ttt$sss" | git stripspace | grep -q "  " ||
-    echo "$ttt$ttt$ttt$sss" | git stripspace | grep -q "  " ||
-    echo "$ttt$sss$sss" | git stripspace | grep -q "  " ||
-    echo "$ttt$ttt$sss$sss" | git stripspace | grep -q "  " ||
-    echo "$ttt$sss$sss$sss" | git stripspace | grep -q "  "
+    ! (echo "$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (echo "$ttt$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (echo "$ttt$ttt$ttt$sss" | git stripspace | grep -q "  ") &&
+    ! (echo "$ttt$sss$sss" | git stripspace | grep -q "  ") &&
+    ! (echo "$ttt$ttt$sss$sss" | git stripspace | grep -q "  ") &&
+    ! (echo "$ttt$sss$sss$sss" | git stripspace | grep -q "  ")
 '
 
 test_expect_success \
@@ -339,13 +339,13 @@ test_expect_success \
     git diff expect actual
 '
 
-test_expect_failure \
+test_expect_success \
     'spaces without newline at end should not show spaces' '
-    printf "" | git stripspace | grep -q " " ||
-    printf "$sss" | git stripspace | grep -q " " ||
-    printf "$sss$sss" | git stripspace | grep -q " " ||
-    printf "$sss$sss$sss" | git stripspace | grep -q " " ||
-    printf "$sss$sss$sss$sss" | git stripspace | grep -q " "
+    ! (printf "" | git stripspace | grep -q " ") &&
+    ! (printf "$sss" | git stripspace | grep -q " ") &&
+    ! (printf "$sss$sss" | git stripspace | grep -q " ") &&
+    ! (printf "$sss$sss$sss" | git stripspace | grep -q " ") &&
+    ! (printf "$sss$sss$sss$sss" | git stripspace | grep -q " ")
 '
 
 test_expect_success \

t/t0040-parse-options.sh

@@ -87,9 +87,9 @@ test_expect_success 'unambiguously abbreviated option with "="' '
 	git diff expect output
 '
 
-test_expect_failure 'ambiguously abbreviated option' '
+test_expect_success 'ambiguously abbreviated option' '
 	test-parse-options --strin 123;
-        test $? != 129
+	test $? = 129
 '
 
 cat > expect << EOF