Просмотр исходного кода

fix(e2e): reject fenced and malformed scenario tables

Filter backtick and tilde fenced examples before locating the E2E scenario-card table, so illustrative Markdown cannot satisfy the mechanical gate or shadow a later real table.\n\nParse only the documented leading/trailing-pipe row form and require a same-width delimiter row whose cells contain at least three hyphens with optional alignment colons. Missing or malformed delimiters now remain check failures instead of being silently treated as data.\n\nThe process-level harness records RED reproductions for both false-pass forms, fenced-only tables, and fenced examples before real tables, then verifies the focused GREEN behavior and prior contracts.
Drew Ritter 1 месяц назад
Родитель
Сommit
2a2e0b41c2

+ 66 - 10
skills/agentic-end-to-end-testing/scripts/check-cards-against-spec

@@ -34,6 +34,35 @@ warn() { echo "warn: $1"; }
 # design spec: markdown re-wrapping must not defeat the verbatim check.)
 # design spec: markdown re-wrapping must not defeat the verbatim check.)
 normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; }
 normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; }
 
 
+# Exclude fenced examples from structural matching. This intentionally models
+# only backtick/tilde fences; it is not a general Markdown parser.
+without_fenced_code() {
+  awk '
+    function fence_family(line, first, count) {
+      sub(/^[[:space:]]*/, "", line)
+      first = substr(line, 1, 1)
+      if (first != "`" && first != "~") return ""
+      count = 0
+      while (substr(line, count + 1, 1) == first) count++
+      return count >= 3 ? first : ""
+    }
+    {
+      marker = fence_family($0)
+      if (!in_fence && marker != "") {
+        in_fence = 1
+        family = marker
+        next
+      }
+      if (in_fence && marker == family) {
+        in_fence = 0
+        family = ""
+        next
+      }
+      if (!in_fence) print
+    }
+  ' "$1"
+}
+
 # Text of the card's Expected section only (case-insensitive heading match,
 # Text of the card's Expected section only (case-insensitive heading match,
 # any ##+ level; section ends at the next heading or EOF).
 # any ##+ level; section ends at the next heading or EOF).
 expected_section() {
 expected_section() {
@@ -56,7 +85,7 @@ TABLE="$(awk '
   }
   }
   insec && /^[[:space:]]*\|/ { intable = 1; print; next }
   insec && /^[[:space:]]*\|/ { intable = 1; print; next }
   insec && intable { exit }
   insec && intable { exit }
-' "$SPEC")"
+' < <(without_fenced_code "$SPEC"))"
 
 
 if [ -z "$TABLE" ]; then
 if [ -z "$TABLE" ]; then
   echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2
   echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2
@@ -68,13 +97,24 @@ fi
 US=$'\x1f'
 US=$'\x1f'
 CARD_COL=-1; FALS_COL=-1; ROWS=0
 CARD_COL=-1; FALS_COL=-1; ROWS=0
 declare -a ROW_CARD ROW_FALS
 declare -a ROW_CARD ROW_FALS
+HEADER_COLS=0; TABLE_VALID=1
 
 
 lineno=0
 lineno=0
 while IFS= read -r line; do
 while IFS= read -r line; do
   lineno=$((lineno + 1))
   lineno=$((lineno + 1))
-  esc="${line//\\|/$US}"
+  row="$(printf '%s' "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  case "$row" in
+    \|*\|) ;;
+    *)
+      fail "row $lineno: canonical table rows require leading and trailing pipes"
+      TABLE_VALID=0
+      break
+      ;;
+  esac
+  row="${row#|}"
+  row="${row%|}"
+  esc="${row//\\|/$US}"
   IFS='|' read -r -a cells <<< "$esc"
   IFS='|' read -r -a cells <<< "$esc"
-  # drop leading/trailing empty fields produced by the outer pipes
   trimmed=()
   trimmed=()
   for c in "${cells[@]}"; do
   for c in "${cells[@]}"; do
     c="${c//$US/|}"
     c="${c//$US/|}"
@@ -88,14 +128,28 @@ while IFS= read -r line; do
       [ "$low" = "card" ] && CARD_COL=$i
       [ "$low" = "card" ] && CARD_COL=$i
       [ "$low" = "falsification" ] && FALS_COL=$i
       [ "$low" = "falsification" ] && FALS_COL=$i
     done
     done
+    HEADER_COLS=${#trimmed[@]}
+    if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then
+      fail "table header must name Card and Falsification columns"
+      TABLE_VALID=0
+      break
+    fi
     continue
     continue
   fi
   fi
-  # separator row: cells of dashes/colons only
-  joined="$(printf '%s' "${trimmed[*]}" | tr -d ' :-')"
-  [ -z "$joined" ] && continue
-  if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then
-    fail "table header must name Card and Falsification columns"
-    break
+  if [ "$lineno" -eq 2 ]; then
+    delimiter_ok=1
+    [ "${#trimmed[@]}" -eq "$HEADER_COLS" ] || delimiter_ok=0
+    for c in "${trimmed[@]}"; do
+      if ! printf '%s\n' "$c" | grep -Eq '^:?-{3,}:?$'; then
+        delimiter_ok=0
+      fi
+    done
+    if [ "$delimiter_ok" -ne 1 ]; then
+      fail "row 2: malformed table delimiter"
+      TABLE_VALID=0
+      break
+    fi
+    continue
   fi
   fi
   card="${trimmed[$CARD_COL]:-}"
   card="${trimmed[$CARD_COL]:-}"
   falsif="${trimmed[$FALS_COL]:-}"
   falsif="${trimmed[$FALS_COL]:-}"
@@ -107,7 +161,9 @@ while IFS= read -r line; do
   ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1))
   ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1))
 done <<< "$TABLE"
 done <<< "$TABLE"
 
 
-[ "$ROWS" -ge 1 ] || fail "scenario table has no data rows"
+if [ "$TABLE_VALID" -eq 1 ] && [ "$ROWS" -lt 1 ]; then
+  fail "scenario table has no data rows"
+fi
 
 
 # --- checks 2-4 per row -----------------------------------------------------
 # --- checks 2-4 per row -----------------------------------------------------
 i=0
 i=0

+ 52 - 0
tests/agentic-e2e-checker/test-check-cards-against-spec.sh

@@ -202,6 +202,58 @@ assert_exit 0 "extra card -> exit 0" \
   "$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards"
   "$CHECKER" "$TEST_ROOT/t6/spec.md" "$TEST_ROOT/t6/cards"
 assert_out_contains "extra-exploration" "warning names the extra card"
 assert_out_contains "extra-exploration" "warning names the extra card"
 
 
+echo "scenario table requires a delimiter row"
+make_spec "$TEST_ROOT/t10"; make_cards "$TEST_ROOT/t10/cards"
+sed -i.bak '/^| --- | --- | --- |$/d' "$TEST_ROOT/t10/spec.md"
+assert_exit 1 "header followed by data without delimiter -> exit 1" \
+  "$CHECKER" "$TEST_ROOT/t10/spec.md" "$TEST_ROOT/t10/cards"
+
+echo "scenario table requires a valid delimiter row"
+make_spec "$TEST_ROOT/t11"; make_cards "$TEST_ROOT/t11/cards"
+sed -i.bak 's/^| --- | --- | --- |$/| -- | --- | --- |/' "$TEST_ROOT/t11/spec.md"
+assert_exit 1 "delimiter cells require at least three hyphens -> exit 1" \
+  "$CHECKER" "$TEST_ROOT/t11/spec.md" "$TEST_ROOT/t11/cards"
+
+echo "scenario table inside a fenced example does not count"
+mkdir -p "$TEST_ROOT/t12/cards"
+make_cards "$TEST_ROOT/t12/cards"
+cat > "$TEST_ROOT/t12/spec.md" <<'EOF'
+# Widget Design
+
+## E2E scenario cards
+
+```markdown
+| Card | Covers | Falsification |
+| --- | --- | --- |
+| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
+| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
+```
+EOF
+assert_exit 2 "fenced-only table -> exit 2" \
+  "$CHECKER" "$TEST_ROOT/t12/spec.md" "$TEST_ROOT/t12/cards"
+
+echo "fenced example before a real table is ignored"
+mkdir -p "$TEST_ROOT/t13/cards"
+make_cards "$TEST_ROOT/t13/cards"
+cat > "$TEST_ROOT/t13/spec.md" <<'EOF'
+# Widget Design
+
+## E2E scenario cards
+
+~~~markdown
+| Card | Covers | Falsification |
+| --- | --- | --- |
+| fake-card | Example only | If the example is absent, the scenario FAILS. |
+~~~
+
+| Card | Covers | Falsification |
+| --- | --- | --- |
+| widget-show-table | Rendered table incl. TOTAL row | If stdout's last line is not `TOTAL` followed by the two-decimal sum (20.85 for the seed fixture), or the TOTAL row is absent entirely, the scenario FAILS. |
+| widget-status-flags | Status output | If `widget status` does not print exactly `OK \| DEGRADED` (a literal pipe) with dots . and stars * intact, the scenario FAILS. |
+EOF
+assert_exit 0 "real table after fenced example -> exit 0" \
+  "$CHECKER" "$TEST_ROOT/t13/spec.md" "$TEST_ROOT/t13/cards"
+
 echo "no scenario table"
 echo "no scenario table"
 mkdir -p "$TEST_ROOT/t7/cards"
 mkdir -p "$TEST_ROOT/t7/cards"
 printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"
 printf '# Widget Design\n\nNo table here.\n' > "$TEST_ROOT/t7/spec.md"