Просмотр исходного кода

fix(tests): stop the SDD skill test flaking on timing and prose case

tests/claude-code/test-subagent-driven-development.sh failed
intermittently for two independent reasons:

- Budget mismatch: the file runs 9 prompts with a 90s timeout each
  (810s worst case) inside the runner's 600s per-file ceiling, so slow
  backend days produced spurious timeouts. Raise the runner default to
  900s and fix the help text, which claimed the default was 300.
- Case-sensitive prose matching: the assert helpers grepped free-form
  model output case-sensitively, but models capitalize the skill's own
  headings — observed failures include "Do Not Trust the Report"
  missing pattern "not trust" and a structured answer missing
  "First:.*spec.*compliance". Match case-insensitively in
  assert_contains/assert_not_contains/assert_count/assert_order, widen
  two Test 5 keyword patterns to phrasings observed in real runs, and
  make assert_order dump the output on failure the way assert_contains
  already does, so the next flake is diagnosable.

Observed 3 failures across 4 runs before the change (timeout, two
distinct pattern misses); 3/3 consecutive full runs pass after it.
Jesse Vincent 1 месяц назад
Родитель
Сommit
cda4b15fd6

+ 3 - 2
tests/claude-code/run-skill-tests.sh

@@ -25,7 +25,8 @@ fi
 # Parse command line arguments
 VERBOSE=false
 SPECIFIC_TEST=""
-TIMEOUT=600  # Default 10 minute timeout per test
+TIMEOUT=900  # Per-test-file budget; must exceed the file's worst case
+             # (test-subagent-driven-development.sh: 9 prompts x 90s each)
 RUN_INTEGRATION=false
 
 while [[ $# -gt 0 ]]; do
@@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do
             echo "Options:"
             echo "  --verbose, -v        Show verbose output"
             echo "  --test, -t NAME      Run only the specified test"
-            echo "  --timeout SECONDS    Set timeout per test (default: 300)"
+            echo "  --timeout SECONDS    Set timeout per test (default: 900)"
             echo "  --integration, -i    Run integration tests (slow, 10-30 min)"
             echo "  --help, -h           Show this help"
             echo ""

+ 11 - 5
tests/claude-code/test-helpers.sh

@@ -30,12 +30,14 @@ run_claude() {
 
 # Check if output contains a pattern
 # Usage: assert_contains "output" "pattern" "test name"
+# Matching is case-insensitive: patterns are prose keywords, and models
+# freely capitalize skill terms ("Do Not Trust", "Spec Compliance").
 assert_contains() {
     local output="$1"
     local pattern="$2"
     local test_name="${3:-test}"
 
-    if echo "$output" | grep -q "$pattern"; then
+    if echo "$output" | grep -qi "$pattern"; then
         echo "  [PASS] $test_name"
         return 0
     else
@@ -54,7 +56,7 @@ assert_not_contains() {
     local pattern="$2"
     local test_name="${3:-test}"
 
-    if echo "$output" | grep -q "$pattern"; then
+    if echo "$output" | grep -qi "$pattern"; then
         echo "  [FAIL] $test_name"
         echo "  Did not expect to find: $pattern"
         echo "  In output:"
@@ -74,7 +76,7 @@ assert_count() {
     local expected="$3"
     local test_name="${4:-test}"
 
-    local actual=$(echo "$output" | grep -c "$pattern" || echo "0")
+    local actual=$(echo "$output" | grep -ci "$pattern" || echo "0")
 
     if [ "$actual" -eq "$expected" ]; then
         echo "  [PASS] $test_name (found $actual instances)"
@@ -98,16 +100,20 @@ assert_order() {
     local test_name="${4:-test}"
 
     # Get line numbers where patterns appear
-    local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1)
-    local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1)
+    local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1)
+    local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1)
 
     if [ -z "$line_a" ]; then
         echo "  [FAIL] $test_name: pattern A not found: $pattern_a"
+        echo "  In output:"
+        echo "$output" | sed 's/^/    /'
         return 1
     fi
 
     if [ -z "$line_b" ]; then
         echo "  [FAIL] $test_name: pattern B not found: $pattern_b"
+        echo "  In output:"
+        echo "$output" | sed 's/^/    /'
         return 1
     fi
 

+ 2 - 2
tests/claude-code/test-subagent-driven-development.sh

@@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..."
 
 output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")
 
-if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
+if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
     : # pass
 else
     exit 1
 fi
 
-if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then
+if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then
     : # pass
 else
     exit 1