test-brainstorm-handoff.sh 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. #!/usr/bin/env bash
  2. # Test: Brainstorm-to-plan handoff
  3. #
  4. # Verifies that after brainstorming, Claude invokes the writing-plans skill
  5. # instead of using EnterPlanMode.
  6. #
  7. # The failure mode this catches:
  8. # User says "build it" after brainstorming -> Claude calls EnterPlanMode
  9. # (because the system prompt's planning guidance overpowers the brainstorming
  10. # skill's instructions, which were loaded many turns ago)
  11. #
  12. # PASS: Skill tool invoked with "writing-plans" AND EnterPlanMode NOT invoked
  13. # FAIL: EnterPlanMode invoked OR writing-plans not invoked
  14. #
  15. # Usage:
  16. # ./test-brainstorm-handoff.sh # Normal test (expects PASS)
  17. # ./test-brainstorm-handoff.sh --without-fix # Strip fix, reproduce failure
  18. # ./test-brainstorm-handoff.sh --verbose # Show full output
  19. #
  20. set -euo pipefail
  21. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  22. PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
  23. # Parse flags
  24. VERBOSE=false
  25. WITHOUT_FIX=false
  26. while [[ $# -gt 0 ]]; do
  27. case $1 in
  28. --verbose|-v) VERBOSE=true; shift ;;
  29. --without-fix) WITHOUT_FIX=true; shift ;;
  30. *) echo "Unknown flag: $1"; exit 1 ;;
  31. esac
  32. done
  33. TIMESTAMP=$(date +%s)
  34. OUTPUT_DIR="/tmp/superpowers-tests/${TIMESTAMP}/brainstorm-handoff"
  35. mkdir -p "$OUTPUT_DIR"
  36. echo "=== Brainstorm-to-Plan Handoff Test ==="
  37. echo "Mode: $([ "$WITHOUT_FIX" = true ] && echo "WITHOUT FIX (expect failure)" || echo "WITH FIX (expect pass)")"
  38. echo "Output: $OUTPUT_DIR"
  39. echo ""
  40. # --- Project Setup ---
  41. PROJECT_DIR="$OUTPUT_DIR/project"
  42. mkdir -p "$PROJECT_DIR/src"
  43. mkdir -p "$PROJECT_DIR/docs/superpowers/specs"
  44. cat > "$PROJECT_DIR/package.json" << 'PROJ_EOF'
  45. {
  46. "name": "my-express-app",
  47. "version": "1.0.0",
  48. "type": "module",
  49. "dependencies": {
  50. "express": "^4.18.0",
  51. "better-sqlite3": "^9.0.0"
  52. }
  53. }
  54. PROJ_EOF
  55. cat > "$PROJECT_DIR/src/index.js" << 'PROJ_EOF'
  56. import express from 'express';
  57. const app = express();
  58. app.use(express.json());
  59. app.get('/health', (req, res) => res.json({ status: 'ok' }));
  60. const PORT = process.env.PORT || 3000;
  61. app.listen(PORT, () => console.log(`Listening on ${PORT}`));
  62. PROJ_EOF
  63. # Pre-create a spec document (simulating completed brainstorming)
  64. cat > "$PROJECT_DIR/docs/superpowers/specs/2025-01-15-url-shortener-design.md" << 'SPEC_EOF'
  65. # URL Shortener Design Spec
  66. ## Overview
  67. Add URL shortening capability to the existing Express.js API.
  68. ## Features
  69. - POST /api/shorten accepts { url } and returns { shortCode, shortUrl }
  70. - GET /:code redirects to the original URL (302)
  71. - GET /api/stats/:code returns { clicks, createdAt, originalUrl }
  72. ## Technical Design
  73. ### Database
  74. Single SQLite table via better-sqlite3:
  75. ```sql
  76. CREATE TABLE urls (
  77. id INTEGER PRIMARY KEY AUTOINCREMENT,
  78. short_code TEXT UNIQUE NOT NULL,
  79. original_url TEXT NOT NULL,
  80. clicks INTEGER DEFAULT 0,
  81. created_at TEXT DEFAULT (datetime('now'))
  82. );
  83. CREATE INDEX idx_short_code ON urls(short_code);
  84. ```
  85. ### File Structure
  86. - `src/index.js` — modified to mount new routes
  87. - `src/db.js` — database initialization and query functions
  88. - `src/shorten.js` — route handlers for all three endpoints
  89. - `src/code-generator.js` — random 6-char alphanumeric code generation
  90. ### Code Generation
  91. Random 6-character alphanumeric codes using crypto.randomBytes.
  92. Check for collisions and retry (astronomically unlikely with 36^6 space).
  93. ### Validation
  94. - URL must be present and start with http:// or https://
  95. - Return 400 with { error: "..." } for invalid input
  96. ### Error Handling
  97. - 404 with { error: "Not found" } for unknown short codes
  98. - 500 with { error: "Internal server error" } for database failures
  99. ## Decisions
  100. - 302 redirects (not 301) so browsers don't cache and we always track clicks
  101. - Database path configurable via DATABASE_PATH env var, defaults to ./data/urls.db
  102. - No auth, no custom codes, no expiry — keeping it simple
  103. SPEC_EOF
  104. # Initialize git so brainstorming can inspect project state
  105. cd "$PROJECT_DIR"
  106. git init -q
  107. git add -A
  108. git commit -q -m "Initial commit with URL shortener spec"
  109. # --- Plugin Setup ---
  110. EFFECTIVE_PLUGIN_DIR="$PLUGIN_DIR"
  111. if [ "$WITHOUT_FIX" = true ]; then
  112. echo "Creating plugin copy without the handoff fix..."
  113. EFFECTIVE_PLUGIN_DIR="$OUTPUT_DIR/plugin-without-fix"
  114. cp -R "$PLUGIN_DIR" "$EFFECTIVE_PLUGIN_DIR"
  115. # Strip fix from brainstorming SKILL.md: revert to old implementation section
  116. python3 << PYEOF
  117. import pathlib
  118. p = pathlib.Path('$EFFECTIVE_PLUGIN_DIR/skills/brainstorming/SKILL.md')
  119. content = p.read_text()
  120. content = content.replace(
  121. '**Implementation (if continuing):**\nWhen the user approves the design and wants to build:\n1. **Invoke \`superpowers:writing-plans\` using the Skill tool.** Not EnterPlanMode. Not plan mode. Not direct implementation. The Skill tool.\n2. After the plan is written, use superpowers:using-git-worktrees to create an isolated workspace for implementation.',
  122. '**Implementation (if continuing):**\n- Ask: "Ready to set up for implementation?"\n- Use superpowers:using-git-worktrees to create isolated workspace\n- **REQUIRED:** Use superpowers:writing-plans to create detailed implementation plan'
  123. )
  124. p.write_text(content)
  125. PYEOF
  126. # Strip fix from using-superpowers: remove EnterPlanMode red flag
  127. python3 << PYEOF
  128. import pathlib
  129. p = pathlib.Path('$EFFECTIVE_PLUGIN_DIR/skills/using-superpowers/SKILL.md')
  130. lines = p.read_text().splitlines(keepends=True)
  131. lines = [l for l in lines if 'I should use EnterPlanMode' not in l]
  132. p.write_text(''.join(lines))
  133. PYEOF
  134. # Strip fix from writing-plans: revert description and context
  135. python3 << PYEOF
  136. import pathlib
  137. p = pathlib.Path('$EFFECTIVE_PLUGIN_DIR/skills/writing-plans/SKILL.md')
  138. content = p.read_text()
  139. content = content.replace(
  140. 'description: Use when you have a spec or requirements for a multi-step task, before touching code. After brainstorming, ALWAYS use this — not EnterPlanMode or plan mode.',
  141. 'description: Use when you have a spec or requirements for a multi-step task, before touching code'
  142. )
  143. content = content.replace(
  144. '**Context:** This runs in the main workspace after brainstorming, while context is fresh. The worktree is created afterward for implementation.',
  145. '**Context:** This should be run in a dedicated worktree (created by brainstorming skill).'
  146. )
  147. p.write_text(content)
  148. PYEOF
  149. echo "Plugin copy created at $EFFECTIVE_PLUGIN_DIR"
  150. echo ""
  151. fi
  152. # --- Run Conversation ---
  153. cd "$PROJECT_DIR"
  154. # Turn 1: Load brainstorming and establish that we finished the design
  155. # The key is that brainstorming gets loaded into context, and we're at the handoff point
  156. echo ">>> Turn 1: Loading brainstorming skill and establishing context..."
  157. TURN1_LOG="$OUTPUT_DIR/turn1.json"
  158. TURN1_PROMPT='I want to add URL shortening to this Express app. I already have the full design worked out and written to docs/superpowers/specs/2025-01-15-url-shortener-design.md. Please read the spec.'
  159. timeout 300 claude -p "$TURN1_PROMPT" \
  160. --plugin-dir "$EFFECTIVE_PLUGIN_DIR" \
  161. --dangerously-skip-permissions \
  162. --max-turns 5 \
  163. --output-format stream-json \
  164. > "$TURN1_LOG" 2>&1 || true
  165. echo "Turn 1 complete."
  166. if [ "$VERBOSE" = true ]; then
  167. echo "---"
  168. grep '"type":"assistant"' "$TURN1_LOG" | tail -1 | jq -r '.message.content[0].text // empty' 2>/dev/null | head -c 800 || true
  169. echo ""
  170. echo "---"
  171. fi
  172. echo ""
  173. # Turn 2: Approve and ask to build - this is the critical handoff moment
  174. echo ">>> Turn 2: 'The spec is done. Build it.' (critical handoff)..."
  175. TURN2_LOG="$OUTPUT_DIR/turn2.json"
  176. TURN2_PROMPT='The spec is complete and I am happy with the design. Build it.'
  177. timeout 300 claude -p "$TURN2_PROMPT" \
  178. --continue \
  179. --plugin-dir "$EFFECTIVE_PLUGIN_DIR" \
  180. --dangerously-skip-permissions \
  181. --max-turns 5 \
  182. --output-format stream-json \
  183. > "$TURN2_LOG" 2>&1 || true
  184. echo "Turn 2 complete."
  185. if [ "$VERBOSE" = true ]; then
  186. echo "---"
  187. grep '"type":"assistant"' "$TURN2_LOG" | tail -1 | jq -r '.message.content[0].text // empty' 2>/dev/null | head -c 800 || true
  188. echo ""
  189. echo "---"
  190. fi
  191. echo ""
  192. # --- Assertions ---
  193. echo "=== Results ==="
  194. echo ""
  195. # Combine all turn logs for analysis
  196. ALL_LOGS="$OUTPUT_DIR/all-turns.json"
  197. cat "$TURN1_LOG" "$TURN2_LOG" > "$ALL_LOGS"
  198. # Detection: writing-plans skill invoked?
  199. HAS_WRITING_PLANS=false
  200. if grep -q '"name":"Skill"' "$ALL_LOGS" 2>/dev/null && grep -q 'writing-plans' "$ALL_LOGS" 2>/dev/null; then
  201. HAS_WRITING_PLANS=true
  202. fi
  203. # Detection: EnterPlanMode invoked?
  204. HAS_ENTER_PLAN_MODE=false
  205. if grep -q '"name":"EnterPlanMode"' "$ALL_LOGS" 2>/dev/null; then
  206. HAS_ENTER_PLAN_MODE=true
  207. fi
  208. # Report what skills were invoked
  209. echo "Skills invoked:"
  210. grep -o '"skill":"[^"]*"' "$ALL_LOGS" 2>/dev/null | sort -u || echo " (none)"
  211. echo ""
  212. echo "Notable tools invoked:"
  213. grep -o '"name":"[A-Z][^"]*"' "$ALL_LOGS" 2>/dev/null | sort | uniq -c | sort -rn | head -10 || echo " (none)"
  214. echo ""
  215. # Determine result
  216. PASSED=false
  217. if [ "$WITHOUT_FIX" = true ]; then
  218. # In without-fix mode, we EXPECT the failure (EnterPlanMode)
  219. echo "--- Without-Fix Mode (reproducing failure) ---"
  220. if [ "$HAS_ENTER_PLAN_MODE" = true ]; then
  221. echo "REPRODUCED: Claude used EnterPlanMode (the bug we're fixing)"
  222. PASSED=true
  223. elif [ "$HAS_WRITING_PLANS" = true ]; then
  224. echo "NOT REPRODUCED: Claude used writing-plans even without the fix"
  225. echo "(The model may have followed the old guidance anyway)"
  226. PASSED=false
  227. else
  228. echo "INCONCLUSIVE: Claude used neither writing-plans nor EnterPlanMode"
  229. echo "The brainstorming flow may not have reached the handoff point."
  230. PASSED=false
  231. fi
  232. else
  233. # Normal mode: expect writing-plans, not EnterPlanMode
  234. echo "--- With-Fix Mode (verifying fix) ---"
  235. if [ "$HAS_WRITING_PLANS" = true ] && [ "$HAS_ENTER_PLAN_MODE" = false ]; then
  236. echo "PASS: Claude used writing-plans skill (correct handoff)"
  237. PASSED=true
  238. elif [ "$HAS_ENTER_PLAN_MODE" = true ]; then
  239. echo "FAIL: Claude used EnterPlanMode instead of writing-plans"
  240. PASSED=false
  241. elif [ "$HAS_WRITING_PLANS" = true ] && [ "$HAS_ENTER_PLAN_MODE" = true ]; then
  242. echo "FAIL: Claude used BOTH writing-plans AND EnterPlanMode"
  243. PASSED=false
  244. else
  245. echo "INCONCLUSIVE: Claude used neither writing-plans nor EnterPlanMode"
  246. echo "The brainstorming flow may not have reached the handoff point."
  247. echo "Check logs - brainstorming may still be asking questions."
  248. PASSED=false
  249. fi
  250. fi
  251. echo ""
  252. # Show the critical turn 2 response
  253. echo "Turn 2 response (first 500 chars):"
  254. grep '"type":"assistant"' "$TURN2_LOG" 2>/dev/null | tail -1 | \
  255. jq -r '.message.content[0].text // .message.content' 2>/dev/null | \
  256. head -c 500 || echo " (could not extract)"
  257. echo ""
  258. echo ""
  259. echo "Logs:"
  260. echo " Turn 1: $TURN1_LOG"
  261. echo " Turn 2: $TURN2_LOG"
  262. echo " Combined: $ALL_LOGS"
  263. echo ""
  264. if [ "$PASSED" = true ]; then
  265. exit 0
  266. else
  267. exit 1
  268. fi