run-claude-describes-sdd.sh 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100
  1. #!/bin/bash
  2. # Test where Claude explicitly describes subagent-driven-development before user requests it
  3. # This mimics the original failure scenario
  4. set -e
  5. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  6. PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
  7. TIMESTAMP=$(date +%s)
  8. OUTPUT_DIR="/tmp/superpowers-tests/${TIMESTAMP}/explicit-skill-requests/claude-describes"
  9. mkdir -p "$OUTPUT_DIR"
  10. PROJECT_DIR="$OUTPUT_DIR/project"
  11. mkdir -p "$PROJECT_DIR/docs/superpowers/plans"
  12. echo "=== Test: Claude Describes SDD First ==="
  13. echo "Output dir: $OUTPUT_DIR"
  14. echo ""
  15. cd "$PROJECT_DIR"
  16. # Create a plan
  17. cat > "$PROJECT_DIR/docs/superpowers/plans/auth-system.md" << 'EOF'
  18. # Auth System Implementation Plan
  19. ## Task 1: Add User Model
  20. Create user model with email and password fields.
  21. ## Task 2: Add Auth Routes
  22. Create login and register endpoints.
  23. ## Task 3: Add JWT Middleware
  24. Protect routes with JWT validation.
  25. EOF
  26. # Turn 1: Have Claude describe execution options including SDD
  27. echo ">>> Turn 1: Ask Claude to describe execution options..."
  28. claude -p "I have a plan at docs/superpowers/plans/auth-system.md. Tell me about my options for executing it, including what subagent-driven-development means and how it works." \
  29. --model haiku \
  30. --plugin-dir "$PLUGIN_DIR" \
  31. --dangerously-skip-permissions \
  32. --max-turns 3 \
  33. --output-format stream-json \
  34. > "$OUTPUT_DIR/turn1.json" 2>&1 || true
  35. echo "Done."
  36. # Turn 2: THE CRITICAL TEST - now that Claude has explained it
  37. echo ">>> Turn 2: Request subagent-driven-development..."
  38. FINAL_LOG="$OUTPUT_DIR/turn2.json"
  39. claude -p "subagent-driven-development, please" \
  40. --continue \
  41. --model haiku \
  42. --plugin-dir "$PLUGIN_DIR" \
  43. --dangerously-skip-permissions \
  44. --max-turns 2 \
  45. --output-format stream-json \
  46. > "$FINAL_LOG" 2>&1 || true
  47. echo "Done."
  48. echo ""
  49. echo "=== Results ==="
  50. # Check Turn 1 to see if Claude described SDD
  51. echo "Turn 1 - Claude's description of options (excerpt):"
  52. grep '"type":"assistant"' "$OUTPUT_DIR/turn1.json" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 800 || echo " (could not extract)"
  53. echo ""
  54. echo "---"
  55. echo ""
  56. # Check final turn
  57. SKILL_PATTERN='"skill":"([^"]*:)?subagent-driven-development"'
  58. if grep -q '"name":"Skill"' "$FINAL_LOG" && grep -qE "$SKILL_PATTERN" "$FINAL_LOG"; then
  59. echo "PASS: Skill was triggered after Claude described it"
  60. TRIGGERED=true
  61. else
  62. echo "FAIL: Skill was NOT triggered (Claude may have thought it already knew)"
  63. TRIGGERED=false
  64. echo ""
  65. echo "Tools invoked in final turn:"
  66. grep '"type":"tool_use"' "$FINAL_LOG" | grep -o '"name":"[^"]*"' | sort -u | head -10 || echo " (none)"
  67. echo ""
  68. echo "Final turn response:"
  69. grep '"type":"assistant"' "$FINAL_LOG" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 800 || echo " (could not extract)"
  70. fi
  71. echo ""
  72. echo "Skills triggered in final turn:"
  73. grep -o '"skill":"[^"]*"' "$FINAL_LOG" 2>/dev/null | sort -u || echo " (none)"
  74. echo ""
  75. echo "Logs in: $OUTPUT_DIR"
  76. if [ "$TRIGGERED" = "true" ]; then
  77. exit 0
  78. else
  79. exit 1
  80. fi