run-multiturn-test.sh 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. #!/bin/bash
  2. # Test explicit skill requests in multi-turn conversations
  3. # Usage: ./run-multiturn-test.sh
  4. #
  5. # This test builds actual conversation history to reproduce the failure mode
  6. # where Claude skips skill invocation after extended conversation
  7. set -e
  8. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  9. PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
  10. TIMESTAMP=$(date +%s)
  11. OUTPUT_DIR="/tmp/superpowers-tests/${TIMESTAMP}/explicit-skill-requests/multiturn"
  12. mkdir -p "$OUTPUT_DIR"
  13. # Create project directory (conversation is cwd-based)
  14. PROJECT_DIR="$OUTPUT_DIR/project"
  15. mkdir -p "$PROJECT_DIR/docs/superpowers/plans"
  16. echo "=== Multi-Turn Explicit Skill Request Test ==="
  17. echo "Output dir: $OUTPUT_DIR"
  18. echo "Project dir: $PROJECT_DIR"
  19. echo "Plugin dir: $PLUGIN_DIR"
  20. echo ""
  21. cd "$PROJECT_DIR"
  22. # Create a dummy plan file
  23. cat > "$PROJECT_DIR/docs/superpowers/plans/auth-system.md" << 'EOF'
  24. # Auth System Implementation Plan
  25. ## Task 1: Add User Model
  26. Create user model with email and password fields.
  27. ## Task 2: Add Auth Routes
  28. Create login and register endpoints.
  29. ## Task 3: Add JWT Middleware
  30. Protect routes with JWT validation.
  31. ## Task 4: Write Tests
  32. Add comprehensive test coverage.
  33. EOF
  34. # Turn 1: Start a planning conversation
  35. echo ">>> Turn 1: Starting planning conversation..."
  36. TURN1_LOG="$OUTPUT_DIR/turn1.json"
  37. claude -p "I need to implement an authentication system. Let's plan this out. The requirements are: user registration with email/password, JWT tokens, and protected routes." \
  38. --plugin-dir "$PLUGIN_DIR" \
  39. --dangerously-skip-permissions \
  40. --max-turns 2 \
  41. --output-format stream-json \
  42. > "$TURN1_LOG" 2>&1 || true
  43. echo "Turn 1 complete."
  44. echo ""
  45. # Turn 2: Continue with more planning detail
  46. echo ">>> Turn 2: Continuing planning..."
  47. TURN2_LOG="$OUTPUT_DIR/turn2.json"
  48. claude -p "Good analysis. I've already written the plan to docs/superpowers/plans/auth-system.md. Now I'm ready to implement. What are my options for execution?" \
  49. --continue \
  50. --plugin-dir "$PLUGIN_DIR" \
  51. --dangerously-skip-permissions \
  52. --max-turns 2 \
  53. --output-format stream-json \
  54. > "$TURN2_LOG" 2>&1 || true
  55. echo "Turn 2 complete."
  56. echo ""
  57. # Turn 3: The critical test - ask for subagent-driven-development
  58. echo ">>> Turn 3: Requesting subagent-driven-development..."
  59. TURN3_LOG="$OUTPUT_DIR/turn3.json"
  60. claude -p "subagent-driven-development, please" \
  61. --continue \
  62. --plugin-dir "$PLUGIN_DIR" \
  63. --dangerously-skip-permissions \
  64. --max-turns 2 \
  65. --output-format stream-json \
  66. > "$TURN3_LOG" 2>&1 || true
  67. echo "Turn 3 complete."
  68. echo ""
  69. echo "=== Results ==="
  70. # Check if skill was triggered in Turn 3
  71. SKILL_PATTERN='"skill":"([^"]*:)?subagent-driven-development"'
  72. if grep -q '"name":"Skill"' "$TURN3_LOG" && grep -qE "$SKILL_PATTERN" "$TURN3_LOG"; then
  73. echo "PASS: Skill 'subagent-driven-development' was triggered in Turn 3"
  74. TRIGGERED=true
  75. else
  76. echo "FAIL: Skill 'subagent-driven-development' was NOT triggered in Turn 3"
  77. TRIGGERED=false
  78. fi
  79. # Show what skills were triggered
  80. echo ""
  81. echo "Skills triggered in Turn 3:"
  82. grep -o '"skill":"[^"]*"' "$TURN3_LOG" 2>/dev/null | sort -u || echo " (none)"
  83. # Check for premature action in Turn 3
  84. echo ""
  85. echo "Checking for premature action in Turn 3..."
  86. FIRST_SKILL_LINE=$(grep -n '"name":"Skill"' "$TURN3_LOG" | head -1 | cut -d: -f1)
  87. if [ -n "$FIRST_SKILL_LINE" ]; then
  88. PREMATURE_TOOLS=$(head -n "$FIRST_SKILL_LINE" "$TURN3_LOG" | \
  89. grep '"type":"tool_use"' | \
  90. grep -v '"name":"Skill"' | \
  91. grep -v '"name":"TodoWrite"' || true)
  92. if [ -n "$PREMATURE_TOOLS" ]; then
  93. echo "WARNING: Tools invoked BEFORE Skill tool in Turn 3:"
  94. echo "$PREMATURE_TOOLS" | head -5
  95. else
  96. echo "OK: No premature tool invocations detected"
  97. fi
  98. else
  99. echo "WARNING: No Skill invocation found in Turn 3"
  100. # Show what WAS invoked
  101. echo ""
  102. echo "Tools invoked in Turn 3:"
  103. grep '"type":"tool_use"' "$TURN3_LOG" | grep -o '"name":"[^"]*"' | head -10 || echo " (none)"
  104. fi
  105. # Show Turn 3 assistant response
  106. echo ""
  107. echo "Turn 3 first assistant response (truncated):"
  108. grep '"type":"assistant"' "$TURN3_LOG" | head -1 | jq -r '.message.content[0].text // .message.content' 2>/dev/null | head -c 500 || echo " (could not extract)"
  109. echo ""
  110. echo "Logs:"
  111. echo " Turn 1: $TURN1_LOG"
  112. echo " Turn 2: $TURN2_LOG"
  113. echo " Turn 3: $TURN3_LOG"
  114. echo "Timestamp: $TIMESTAMP"
  115. if [ "$TRIGGERED" = "true" ]; then
  116. exit 0
  117. else
  118. exit 1
  119. fi