mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-13 07:53:04 +08:00
76803d789a
Adds comprehensive eval infrastructure: - Tier 1 (free): 13 new static tests — cross-skill path consistency, QA structure validation, greptile format, planted-bug fixture validation - Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo, 3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs) - Tier 3 (LLM judge): QA workflow quality, health rubric clarity, cross-skill consistency, baseline score pinning New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON, review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY). Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks. `bun run test:evals` runs everything that costs money (~$4/run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
44 lines
1.6 KiB
JSON
44 lines
1.6 KiB
JSON
{
|
|
"fixture": "qa-eval-spa.html",
|
|
"bugs": [
|
|
{
|
|
"id": "broken-route",
|
|
"category": "functional",
|
|
"severity": "high",
|
|
"description": "Products nav link points to #/prodcts (typo) instead of #/products — shows 'Page not found'",
|
|
"detection_hint": "route|prodcts|typo|products|not found|broken link|navigation"
|
|
},
|
|
{
|
|
"id": "stale-cart-state",
|
|
"category": "functional",
|
|
"severity": "medium",
|
|
"description": "Cart count persists across route changes — never resets when navigating away from products",
|
|
"detection_hint": "cart|count|state|persist|reset|stale|navigation"
|
|
},
|
|
{
|
|
"id": "async-fetch-error",
|
|
"category": "functional",
|
|
"severity": "high",
|
|
"description": "Product list briefly loads then shows 'Error: Failed to fetch products from API' after 1 second",
|
|
"detection_hint": "error|fetch|products|API|loading|failed|async"
|
|
},
|
|
{
|
|
"id": "missing-aria-current",
|
|
"category": "accessibility",
|
|
"severity": "medium",
|
|
"description": "Navigation links have no aria-current attribute to indicate the active route",
|
|
"detection_hint": "aria|current|active|navigation|accessibility|a11y"
|
|
},
|
|
{
|
|
"id": "console-warn-leak",
|
|
"category": "console",
|
|
"severity": "medium",
|
|
"description": "console.warn fires on every route change: 'Possible memory leak detected: 11 event listeners'",
|
|
"detection_hint": "console|warn|memory leak|listener|event|warning"
|
|
}
|
|
],
|
|
"total_bugs": 5,
|
|
"minimum_detection": 3,
|
|
"max_false_positives": 2
|
|
}
|