test-document-review-system.sh 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172
  1. #!/usr/bin/env bash
  2. # Test: Document Review System
  3. # Verifies that spec and plan document reviewers are integrated correctly
  4. set -euo pipefail
  5. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
  6. source "$SCRIPT_DIR/test-helpers.sh"
  7. echo "=== Test: Document Review System ==="
  8. echo ""
  9. # Test 1: Spec document reviewer exists and describes correct checks
  10. echo "Test 1: Spec document reviewer checks..."
  11. output=$(run_claude "What does the spec document reviewer check for in the brainstorming skill? List the categories." 30)
  12. if assert_contains "$output" "Completeness\|completeness" "Checks completeness"; then
  13. : # pass
  14. else
  15. exit 1
  16. fi
  17. if assert_contains "$output" "TODO\|placeholder" "Checks for TODOs"; then
  18. : # pass
  19. else
  20. exit 1
  21. fi
  22. echo ""
  23. # Test 2: Brainstorming skill has spec review loop
  24. echo "Test 2: Brainstorming skill spec review loop..."
  25. output=$(run_claude "Does the brainstorming skill have a spec review loop? What happens if issues are found?" 30)
  26. if assert_contains "$output" "review.*loop\|loop.*review\|re-dispatch\|repeat\|re-review" "Has review loop"; then
  27. : # pass
  28. else
  29. exit 1
  30. fi
  31. if assert_contains "$output" "fix.*issues\|issues.*fix" "Fix issues mentioned"; then
  32. : # pass
  33. else
  34. exit 1
  35. fi
  36. echo ""
  37. # Test 3: Plan document reviewer exists and checks correct things
  38. echo "Test 3: Plan document reviewer checks..."
  39. output=$(run_claude "What does the plan document reviewer check for in the writing-plans skill? What categories?" 30)
  40. if assert_contains "$output" "Spec Alignment\|spec alignment\|matches.*spec" "Checks spec alignment"; then
  41. : # pass
  42. else
  43. exit 1
  44. fi
  45. if assert_contains "$output" "Task Decomposition\|task decomposition\|atomic" "Checks task decomposition"; then
  46. : # pass
  47. else
  48. exit 1
  49. fi
  50. echo ""
  51. # Test 4: Writing-plans skill has chunk-by-chunk review
  52. echo "Test 4: Chunk-by-chunk plan review..."
  53. output=$(run_claude "How does the writing-plans skill review plans? Is it all at once or chunk by chunk?" 30)
  54. if assert_contains "$output" "chunk" "Mentions chunks"; then
  55. : # pass
  56. else
  57. exit 1
  58. fi
  59. if assert_contains "$output" "1000.*line\|under.*1000\|≤1000" "Mentions chunk size limit"; then
  60. : # pass
  61. else
  62. exit 1
  63. fi
  64. echo ""
  65. # Test 5: Review loops have iteration guidance
  66. echo "Test 5: Review loop iteration guidance..."
  67. output=$(run_claude "In the brainstorming or writing-plans skills, what happens if the review loop runs too many times? Is there a limit?" 30)
  68. if assert_contains "$output" "5.*iteration\|5 iteration\|exceed.*5\|human.*guidance" "Has iteration limit or escalation"; then
  69. : # pass
  70. else
  71. exit 1
  72. fi
  73. echo ""
  74. # Test 6: Checkbox syntax is on steps only
  75. echo "Test 6: Checkbox syntax on steps..."
  76. output=$(run_claude "In writing-plans, where should checkbox syntax be used - on task headings, steps, or both?" 30)
  77. if assert_contains "$output" "step" "Mentions steps"; then
  78. : # pass
  79. else
  80. exit 1
  81. fi
  82. if assert_not_contains "$output" "task.*heading.*checkbox\|checkbox.*task.*heading" "Not on task headings"; then
  83. : # pass
  84. else
  85. exit 1
  86. fi
  87. echo ""
  88. # Test 7: Specs go to correct directory
  89. echo "Test 7: Spec document directory..."
  90. output=$(run_claude "Where does the brainstorming skill save spec documents? What directory?" 30)
  91. if assert_contains "$output" "docs/superpowers/specs" "Uses correct spec directory"; then
  92. : # pass
  93. else
  94. exit 1
  95. fi
  96. echo ""
  97. # Test 8: Plans go to correct directory
  98. echo "Test 8: Plan document directory..."
  99. output=$(run_claude "Where does the writing-plans skill save plan documents? What directory?" 30)
  100. if assert_contains "$output" "docs/superpowers/plans" "Uses correct plan directory"; then
  101. : # pass
  102. else
  103. exit 1
  104. fi
  105. echo ""
  106. # Test 9: Reviewers are advisory
  107. echo "Test 9: Reviewer advisory nature..."
  108. output=$(run_claude "Are the spec and plan document reviewers blocking or advisory? Can disagreements be explained?" 30)
  109. if assert_contains "$output" "advisory\|explain.*disagreement\|disagreement" "Reviewers are advisory"; then
  110. : # pass
  111. else
  112. exit 1
  113. fi
  114. echo ""
  115. # Test 10: Same agent fixes issues (preserves context)
  116. echo "Test 10: Same agent fixes issues..."
  117. output=$(run_claude "In the document review loops, who fixes the issues - a new agent or the same agent that wrote the document?" 30)
  118. if assert_contains "$output" "same.*agent\|preserves.*context\|same agent" "Same agent fixes issues"; then
  119. : # pass
  120. else
  121. exit 1
  122. fi
  123. echo ""
  124. echo "=== All document review system tests passed ==="