test-subagent-driven-development-integration.sh 10.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. #!/usr/bin/env bash
  2. # Integration Test: subagent-driven-development workflow
  3. # Actually executes a plan and verifies the new workflow behaviors
  4. set -euo pipefail
  5. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
  6. source "$SCRIPT_DIR/test-helpers.sh"
  7. echo "========================================"
  8. echo " Integration Test: subagent-driven-development"
  9. echo "========================================"
  10. echo ""
  11. echo "This test executes a real plan using the skill and verifies:"
  12. echo " 1. Plan is read once (not per task)"
  13. echo " 2. Full task text provided to subagents"
  14. echo " 3. Subagents perform self-review"
  15. echo " 4. Spec compliance review before code quality"
  16. echo " 5. Review loops when issues found"
  17. echo " 6. Spec reviewer reads code independently"
  18. echo ""
  19. echo "WARNING: This test may take 10-30 minutes to complete."
  20. echo ""
  21. # Create test project
  22. TEST_PROJECT=$(create_test_project)
  23. echo "Test project: $TEST_PROJECT"
  24. # Trap to cleanup
  25. trap "cleanup_test_project $TEST_PROJECT" EXIT
  26. # Set up minimal Node.js project
  27. cd "$TEST_PROJECT"
  28. cat > package.json <<'EOF'
  29. {
  30. "name": "test-project",
  31. "version": "1.0.0",
  32. "type": "module",
  33. "scripts": {
  34. "test": "node --test"
  35. }
  36. }
  37. EOF
  38. mkdir -p src test docs/plans
  39. # Create a simple implementation plan
  40. cat > docs/plans/implementation-plan.md <<'EOF'
  41. # Test Implementation Plan
  42. This is a minimal plan to test the subagent-driven-development workflow.
  43. ## Task 1: Create Add Function
  44. Create a function that adds two numbers.
  45. **File:** `src/math.js`
  46. **Requirements:**
  47. - Function named `add`
  48. - Takes two parameters: `a` and `b`
  49. - Returns the sum of `a` and `b`
  50. - Export the function
  51. **Implementation:**
  52. ```javascript
  53. export function add(a, b) {
  54. return a + b;
  55. }
  56. ```
  57. **Tests:** Create `test/math.test.js` that verifies:
  58. - `add(2, 3)` returns `5`
  59. - `add(0, 0)` returns `0`
  60. - `add(-1, 1)` returns `0`
  61. **Verification:** `npm test`
  62. ## Task 2: Create Multiply Function
  63. Create a function that multiplies two numbers.
  64. **File:** `src/math.js` (add to existing file)
  65. **Requirements:**
  66. - Function named `multiply`
  67. - Takes two parameters: `a` and `b`
  68. - Returns the product of `a` and `b`
  69. - Export the function
  70. - DO NOT add any extra features (like power, divide, etc.)
  71. **Implementation:**
  72. ```javascript
  73. export function multiply(a, b) {
  74. return a * b;
  75. }
  76. ```
  77. **Tests:** Add to `test/math.test.js`:
  78. - `multiply(2, 3)` returns `6`
  79. - `multiply(0, 5)` returns `0`
  80. - `multiply(-2, 3)` returns `-6`
  81. **Verification:** `npm test`
  82. EOF
  83. # Initialize git repo
  84. git init --quiet
  85. git config user.email "test@test.com"
  86. git config user.name "Test User"
  87. git add .
  88. git commit -m "Initial commit" --quiet
  89. echo ""
  90. echo "Project setup complete. Starting execution..."
  91. echo ""
  92. # Run Claude with subagent-driven-development
  93. # Capture full output to analyze
  94. OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"
  95. # Create prompt file
  96. cat > "$TEST_PROJECT/prompt.txt" <<'EOF'
  97. I want you to execute the implementation plan at docs/plans/implementation-plan.md using the subagent-driven-development skill.
  98. IMPORTANT: Follow the skill exactly. I will be verifying that you:
  99. 1. Read the plan once at the beginning
  100. 2. Provide full task text to subagents (don't make them read files)
  101. 3. Ensure subagents do self-review before reporting
  102. 4. Run spec compliance review before code quality review
  103. 5. Use review loops when issues are found
  104. Begin now. Execute the plan.
  105. EOF
  106. # Note: We use a longer timeout since this is integration testing
  107. PROMPT=$(cat "$TEST_PROJECT/prompt.txt")
  108. timeout 1800 claude -p "$PROMPT" > "$OUTPUT_FILE" 2>&1 || {
  109. echo "EXECUTION FAILED"
  110. cat "$OUTPUT_FILE"
  111. exit 1
  112. }
  113. echo ""
  114. echo "Execution complete. Analyzing results..."
  115. echo ""
  116. # Read the output
  117. OUTPUT=$(cat "$OUTPUT_FILE")
  118. # Verification tests
  119. FAILED=0
  120. echo "=== Verification Tests ==="
  121. echo ""
  122. # Test 1: Plan should be read once at the beginning
  123. echo "Test 1: Plan read once at beginning..."
  124. if echo "$OUTPUT" | grep -q "Load Plan\|read.*plan\|extract.*tasks"; then
  125. # Check it's near the beginning (within first 20% of output)
  126. total_lines=$(echo "$OUTPUT" | wc -l)
  127. plan_line=$(echo "$OUTPUT" | grep -n "Load Plan\|read.*plan\|extract.*tasks" | head -1 | cut -d: -f1)
  128. threshold=$((total_lines / 5))
  129. if [ "$plan_line" -lt "$threshold" ]; then
  130. echo " [PASS] Plan read early (line $plan_line of $total_lines)"
  131. else
  132. echo " [FAIL] Plan not read early (line $plan_line of $total_lines)"
  133. FAILED=$((FAILED + 1))
  134. fi
  135. # Should NOT re-read for each task
  136. read_count=$(echo "$OUTPUT" | grep -c "read.*plan" || echo "0")
  137. if [ "$read_count" -le 3 ]; then # Allow some mentions but not per-task
  138. echo " [PASS] Plan not re-read per task ($read_count mentions)"
  139. else
  140. echo " [FAIL] Plan read too many times ($read_count mentions)"
  141. FAILED=$((FAILED + 1))
  142. fi
  143. else
  144. echo " [FAIL] No evidence of plan loading"
  145. FAILED=$((FAILED + 1))
  146. fi
  147. echo ""
  148. # Test 2: Full task text provided to subagents
  149. echo "Test 2: Full task text provided to subagents..."
  150. if echo "$OUTPUT" | grep -q "Task Description.*Requirements\|FULL TEXT"; then
  151. echo " [PASS] Task text appears to be provided in prompts"
  152. else
  153. echo " [FAIL] No evidence of full task text in prompts"
  154. FAILED=$((FAILED + 1))
  155. fi
  156. # Should NOT make subagent read files
  157. if echo "$OUTPUT" | grep -q "Read.*docs/plans/implementation-plan.md" | grep -v "I'm reading\|I read"; then
  158. echo " [FAIL] Subagent was told to read plan file"
  159. FAILED=$((FAILED + 1))
  160. else
  161. echo " [PASS] Subagent not told to read plan file"
  162. fi
  163. echo ""
  164. # Test 3: Subagents do self-review
  165. echo "Test 3: Subagents perform self-review..."
  166. if echo "$OUTPUT" | grep -qi "self-review\|self review\|reviewing my work\|look.*with fresh eyes"; then
  167. echo " [PASS] Self-review mentioned"
  168. # Check for self-review findings
  169. if echo "$OUTPUT" | grep -qi "completeness\|quality\|discipline"; then
  170. echo " [PASS] Self-review checklist items mentioned"
  171. else
  172. echo " [WARN] Self-review checklist items not clearly mentioned"
  173. fi
  174. else
  175. echo " [FAIL] No evidence of self-review"
  176. FAILED=$((FAILED + 1))
  177. fi
  178. echo ""
  179. # Test 4: Spec compliance review before code quality
  180. echo "Test 4: Spec compliance review before code quality..."
  181. spec_line=$(echo "$OUTPUT" | grep -ni "spec.*compliance.*review" | head -1 | cut -d: -f1)
  182. code_line=$(echo "$OUTPUT" | grep -ni "code.*quality.*review" | head -1 | cut -d: -f1)
  183. if [ -n "$spec_line" ] && [ -n "$code_line" ]; then
  184. if [ "$spec_line" -lt "$code_line" ]; then
  185. echo " [PASS] Spec compliance review before code quality (line $spec_line < $code_line)"
  186. else
  187. echo " [FAIL] Code quality before spec compliance (line $code_line < $spec_line)"
  188. FAILED=$((FAILED + 1))
  189. fi
  190. else
  191. if [ -z "$spec_line" ]; then
  192. echo " [FAIL] No spec compliance review found"
  193. FAILED=$((FAILED + 1))
  194. fi
  195. if [ -z "$code_line" ]; then
  196. echo " [WARN] No code quality review found (might be acceptable if spec review caught issues)"
  197. fi
  198. fi
  199. echo ""
  200. # Test 5: Spec reviewer is skeptical and reads code
  201. echo "Test 5: Spec reviewer reads code independently..."
  202. if echo "$OUTPUT" | grep -qi "do not trust.*report\|verify.*independently\|reading.*code\|inspecting.*implementation"; then
  203. echo " [PASS] Spec reviewer reads code independently"
  204. else
  205. echo " [FAIL] No evidence of independent code verification"
  206. FAILED=$((FAILED + 1))
  207. fi
  208. echo ""
  209. # Test 6: Implementation actually works
  210. echo "Test 6: Implementation verification..."
  211. if [ -f "$TEST_PROJECT/src/math.js" ]; then
  212. echo " [PASS] src/math.js created"
  213. if grep -q "export function add" "$TEST_PROJECT/src/math.js"; then
  214. echo " [PASS] add function exists"
  215. else
  216. echo " [FAIL] add function missing"
  217. FAILED=$((FAILED + 1))
  218. fi
  219. if grep -q "export function multiply" "$TEST_PROJECT/src/math.js"; then
  220. echo " [PASS] multiply function exists"
  221. else
  222. echo " [FAIL] multiply function missing"
  223. FAILED=$((FAILED + 1))
  224. fi
  225. else
  226. echo " [FAIL] src/math.js not created"
  227. FAILED=$((FAILED + 1))
  228. fi
  229. if [ -f "$TEST_PROJECT/test/math.test.js" ]; then
  230. echo " [PASS] test/math.test.js created"
  231. else
  232. echo " [FAIL] test/math.test.js not created"
  233. FAILED=$((FAILED + 1))
  234. fi
  235. # Try running tests
  236. if cd "$TEST_PROJECT" && npm test > test-output.txt 2>&1; then
  237. echo " [PASS] Tests pass"
  238. else
  239. echo " [FAIL] Tests failed"
  240. cat test-output.txt
  241. FAILED=$((FAILED + 1))
  242. fi
  243. echo ""
  244. # Test 7: Git commits show proper workflow
  245. echo "Test 7: Git commit history..."
  246. commit_count=$(git -C "$TEST_PROJECT" log --oneline | wc -l)
  247. if [ "$commit_count" -gt 2 ]; then # Initial + at least 2 task commits
  248. echo " [PASS] Multiple commits created ($commit_count total)"
  249. else
  250. echo " [FAIL] Too few commits ($commit_count, expected >2)"
  251. FAILED=$((FAILED + 1))
  252. fi
  253. echo ""
  254. # Test 8: Check for extra features (spec compliance should catch)
  255. echo "Test 8: No extra features added (spec compliance)..."
  256. if grep -q "export function divide\|export function power\|export function subtract" "$TEST_PROJECT/src/math.js" 2>/dev/null; then
  257. echo " [WARN] Extra features found (spec review should have caught this)"
  258. # Not failing on this as it tests reviewer effectiveness
  259. else
  260. echo " [PASS] No extra features added"
  261. fi
  262. echo ""
  263. # Summary
  264. echo "========================================"
  265. echo " Test Summary"
  266. echo "========================================"
  267. echo ""
  268. if [ $FAILED -eq 0 ]; then
  269. echo "STATUS: PASSED"
  270. echo "All verification tests passed!"
  271. echo ""
  272. echo "The subagent-driven-development skill correctly:"
  273. echo " ✓ Reads plan once at start"
  274. echo " ✓ Provides full task text to subagents"
  275. echo " ✓ Enforces self-review"
  276. echo " ✓ Runs spec compliance before code quality"
  277. echo " ✓ Spec reviewer verifies independently"
  278. echo " ✓ Produces working implementation"
  279. exit 0
  280. else
  281. echo "STATUS: FAILED"
  282. echo "Failed $FAILED verification tests"
  283. echo ""
  284. echo "Output saved to: $OUTPUT_FILE"
  285. echo ""
  286. echo "Review the output to see what went wrong."
  287. exit 1
  288. fi