Add summary for changed snapshots and failing tests in test outputs

toolproof/src/main.rs

@@ -975,7 +975,7 @@ async fn main_inner() -> Result<(), ()> {
             _ => None,
         })
         .collect::<Vec<_>>();
-    let mut resolved_errors = 0;
+    let mut resolved: Vec<String> = vec![];
 
     println!("\n{}\n", "Finished running tests".bold());
 
@@ -1008,7 +1008,7 @@ async fn main_inner() -> Result<(), ()> {
                     HoldingError::TestFailure => {}
                     HoldingError::SnapFailure { out } => {
                         if confirm_snapshot(&term, &file, &out).is_ok_and(|v| v) {
-                            resolved_errors += 1;
+                            resolved.push(file.file_path.clone());
 
                             if let Err(e) = tokio::fs::write(&file.file_path, out).await {
                                 eprintln!("Unable to write updates snapshot to disk.\n{e}");
@@ -1033,12 +1033,31 @@ async fn main_inner() -> Result<(), ()> {
         )
     };
 
-    let failing = results.iter().filter(|r| r.is_err()).count() - resolved_errors;
+    let hard_failures = results
+        .iter()
+        .filter_map(|r| match r {
+            Err((file, HoldingError::TestFailure)) => Some(file),
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    let changed_snapshots = results
+        .iter()
+        .filter_map(|r| match r {
+            Err((file, HoldingError::SnapFailure { .. }))
+                if !resolved.contains(&file.file_path) =>
+            {
+                Some(file)
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+
+    let failing = hard_failures.len() + changed_snapshots.len();
     let passing = results
         .iter()
         .filter(|r| matches!(r, Ok(ToolproofTestSuccess::Passed { .. })))
         .count()
-        + resolved_errors;
+        + resolved.len();
     let skipped = results
         .iter()
         .filter(|r| matches!(r, Ok(ToolproofTestSuccess::Skipped)))
@@ -1054,13 +1073,42 @@ async fn main_inner() -> Result<(), ()> {
     };
 
     println!(
-        "{}\n{}\n{}\n{}",
+        "{}\n{}\n{}\n{}\n{}",
         style(&format!("Total passing tests: {}", passing)).cyan(),
         style(&format!("Passed after retry: {}", retried_passed)).cyan(),
-        style(&format!("Failing tests: {}", failing)).cyan(),
+        style(&format!("Failing tests: {}", hard_failures.len())).cyan(),
+        style(&format!("Changed snapshots: {}", changed_snapshots.len())).cyan(),
         style(&format!("Skipped tests: {}", skipped)).cyan(),
     );
 
+    let shell_quote = |s: &str| format!("'{}'", s.replace('\'', "'\\''"));
+
+    if !hard_failures.is_empty() {
+        println!("\n{}", "Failing tests:".red().bold());
+        for file in &hard_failures {
+            println!("  {} {}", "✘".red().bold(), file.name.red());
+        }
+        println!("\n{}", "Rerun the failing tests with:".bold());
+        for file in &hard_failures {
+            println!("  toolproof --name {}", shell_quote(&file.name));
+        }
+    }
+
+    if !changed_snapshots.is_empty() {
+        println!("\n{}", "Changed snapshots:".yellow().bold());
+        for file in &changed_snapshots {
+            println!("  {} {}", "⚠".yellow().bold(), file.name.yellow());
+        }
+        println!(
+            "\n{}",
+            "Review the changed snapshots. If a change is correct, accept it with --update (-u):"
+                .bold()
+        );
+        for file in &changed_snapshots {
+            println!("  toolproof --name {} --update", shell_quote(&file.name));
+        }
+    }
+
     if failing > 0 {
         println!(
             "{}",

toolproof/test_suite/base/default_placeholders.toolproof.yml

@@ -23,6 +23,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/base/extract.toolproof.yml

@@ -25,6 +25,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/base/snapshot.toolproof.yml

@@ -23,6 +23,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed
@@ -63,7 +64,14 @@ steps:
       ╎
       ╎Total passing tests: 0
       ╎Passed after retry: 0
-      ╎Failing tests: 1
+      ╎Failing tests: 0
+      ╎Changed snapshots: 1
       ╎Skipped tests: 0
       ╎
+      ╎Changed snapshots:
+      ╎  ⚠ Inner failing snapshot test
+      ╎
+      ╎Review the changed snapshots. If a change is correct, accept it with --update (-u):
+      ╎  toolproof --name 'Inner failing snapshot test' --update
+      ╎
       ╎Some tests failed

toolproof/test_suite/base/snapshot_file.toolproof.yml

@@ -31,6 +31,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/base/snapshot_update.toolproof.yml

@@ -23,6 +23,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/base/step_platforms.toolproof.yml

@@ -26,6 +26,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/base/summary_mixed_failures.toolproof.yml

@@ -0,0 +1,80 @@
+name: Summary separates snapshot changes from failures
+
+steps:
+  - step: I have a "normal.toolproof.yml" file with the content {yaml}
+    yaml: |-
+      name: Normal failing test
+
+      steps:
+        - I run 'echo "hello"'
+        - stdout should contain "goodbye"
+  - step: I have a "snap.toolproof.yml" file with the content {yaml}
+    yaml: |-
+      name: Snapshot changing test
+
+      steps:
+        - I run 'echo "actual output"'
+        - snapshot: stdout
+          snapshot_content: |-
+            ╎stale output
+  - I run "%toolproof_path% --porcelain -c 1" and expect it to fail
+  - snapshot: stdout
+    snapshot_content: |-
+      ╎
+      ╎Running tests
+      ╎
+      ╎✘ Normal failing test
+      ╎--- STEPS ---
+      ╎✓ I run 'echo "hello"'
+      ╎✘ stdout should contain "goodbye"
+      ╎--- ERROR ---
+      ╎Error in step "stdout should contain "goodbye"":
+      ╎--
+      ╎Failed assertion: The value
+      ╎---
+      ╎"hello\n"
+      ╎---
+      ╎does not contain
+      ╎---
+      ╎"goodbye"
+      ╎---
+      ╎⚠ Snapshot changing test
+      ╎--- SNAPSHOT CHANGED ---
+      ╎
+      ╎    0     0 │ name: Snapshot changing test
+      ╎    1     1 │ 
+      ╎    2     2 │ steps:
+      ╎    3     3 │   - I run 'echo "actual output"'
+      ╎    4     4 │   - snapshot: stdout
+      ╎    5     5 │     snapshot_content: |-
+      ╎    6       │-      ╎stale output
+      ╎          6 │+      ╎actual output
+      ╎
+      ╎
+      ╎--- END SNAPSHOT CHANGE ---
+      ╎
+      ╎Run in interactive mode (-i) or with --update (-u) to accept new snapshots
+      ╎
+      ╎
+      ╎Finished running tests
+      ╎
+      ╎Total passing tests: 0
+      ╎Passed after retry: 0
+      ╎Failing tests: 1
+      ╎Changed snapshots: 1
+      ╎Skipped tests: 0
+      ╎
+      ╎Failing tests:
+      ╎  ✘ Normal failing test
+      ╎
+      ╎Rerun the failing tests with:
+      ╎  toolproof --name 'Normal failing test'
+      ╎
+      ╎Changed snapshots:
+      ╎  ⚠ Snapshot changing test
+      ╎
+      ╎Review the changed snapshots. If a change is correct, accept it with --update (-u):
+      ╎  toolproof --name 'Snapshot changing test' --update
+      ╎
+      ╎Some tests failed
+  - stderr should be empty

toolproof/test_suite/base/summary_multiple_failures.toolproof.yml

@@ -0,0 +1,72 @@
+name: Summary lists every failing test
+
+steps:
+  - step: I have a "alpha.toolproof.yml" file with the content {yaml}
+    yaml: |-
+      name: Alpha failing test
+
+      steps:
+        - I run 'echo "hello"'
+        - stdout should contain "goodbye"
+  - step: I have a "beta.toolproof.yml" file with the content {yaml}
+    yaml: |-
+      name: Beta failing test
+
+      steps:
+        - I run 'echo "hello"'
+        - stdout should contain "farewell"
+  - I run "%toolproof_path% --porcelain -c 1" and expect it to fail
+  - snapshot: stdout
+    snapshot_content: |-
+      ╎
+      ╎Running tests
+      ╎
+      ╎✘ Alpha failing test
+      ╎--- STEPS ---
+      ╎✓ I run 'echo "hello"'
+      ╎✘ stdout should contain "goodbye"
+      ╎--- ERROR ---
+      ╎Error in step "stdout should contain "goodbye"":
+      ╎--
+      ╎Failed assertion: The value
+      ╎---
+      ╎"hello\n"
+      ╎---
+      ╎does not contain
+      ╎---
+      ╎"goodbye"
+      ╎---
+      ╎✘ Beta failing test
+      ╎--- STEPS ---
+      ╎✓ I run 'echo "hello"'
+      ╎✘ stdout should contain "farewell"
+      ╎--- ERROR ---
+      ╎Error in step "stdout should contain "farewell"":
+      ╎--
+      ╎Failed assertion: The value
+      ╎---
+      ╎"hello\n"
+      ╎---
+      ╎does not contain
+      ╎---
+      ╎"farewell"
+      ╎---
+      ╎
+      ╎Finished running tests
+      ╎
+      ╎Total passing tests: 0
+      ╎Passed after retry: 0
+      ╎Failing tests: 2
+      ╎Changed snapshots: 0
+      ╎Skipped tests: 0
+      ╎
+      ╎Failing tests:
+      ╎  ✘ Alpha failing test
+      ╎  ✘ Beta failing test
+      ╎
+      ╎Rerun the failing tests with:
+      ╎  toolproof --name 'Alpha failing test'
+      ╎  toolproof --name 'Beta failing test'
+      ╎
+      ╎Some tests failed
+  - stderr should be empty

toolproof/test_suite/base/test_platforms.toolproof.yml

@@ -64,6 +64,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 2
       ╎
       ╎All tests passed

toolproof/test_suite/browser/selectors.toolproof.yml

@@ -100,6 +100,13 @@ steps:
       ╎Total passing tests: 0
       ╎Passed after retry: 0
       ╎Failing tests: 1
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
+      ╎Failing tests:
+      ╎  ✘ Inner failing test
+      ╎
+      ╎Rerun the failing tests with:
+      ╎  toolproof --name 'Inner failing test'
+      ╎
       ╎Some tests failed

toolproof/test_suite/browser/snapshot.toolproof.yml

@@ -46,6 +46,6 @@ steps:
             ╎Hello World
   - I run "%toolproof_path% --timeout 60" and expect it to fail
   - step: "stdout should contain 'Total passing tests: 0'"
-  - step: "stdout should contain 'Failing tests: 1'"
+  - step: "stdout should contain 'Changed snapshots: 1'"
   - step: "stdout should contain 'Some tests failed'"
   - stderr should be empty

toolproof/test_suite/browser/wait_for.toolproof.yml

@@ -76,6 +76,13 @@ steps:
       ╎Total passing tests: 0
       ╎Passed after retry: 0
       ╎Failing tests: 1
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
+      ╎Failing tests:
+      ╎  ✘ Inner failing test
+      ╎
+      ╎Rerun the failing tests with:
+      ╎  toolproof --name 'Inner failing test'
+      ╎
       ╎Some tests failed

toolproof/test_suite/hooks/before_all.toolproof.yml

@@ -27,6 +27,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/hooks/macros.toolproof.yml

@@ -33,6 +33,7 @@ steps:
       ╎Total passing tests: 1
       ╎Passed after retry: 0
       ╎Failing tests: 0
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
       ╎All tests passed

toolproof/test_suite/refs/ref-file.toolproof.yml

@@ -78,7 +78,14 @@ steps:
       ╎Total passing tests: 0
       ╎Passed after retry: 0
       ╎Failing tests: 1
+      ╎Changed snapshots: 0
       ╎Skipped tests: 0
       ╎
+      ╎Failing tests:
+      ╎  ✘ Inner failing test
+      ╎
+      ╎Rerun the failing tests with:
+      ╎  toolproof --name 'Inner failing test'
+      ╎
       ╎Some tests failed
   - stderr should be empty