test-bootstrap-caching.mjs 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268
  1. import fs from 'fs';
  2. import { pathToFileURL } from 'url';
  3. const [, , pluginPath, scenario] = process.argv;
  4. if (!pluginPath || !['present', 'missing'].includes(scenario)) {
  5. console.error('Usage: node test-bootstrap-caching.mjs PLUGIN_PATH present|missing');
  6. process.exit(2);
  7. }
  8. let existsCount = 0;
  9. let readCount = 0;
  10. const originalExistsSync = fs.existsSync;
  11. const originalReadFileSync = fs.readFileSync;
  12. fs.existsSync = function (...args) {
  13. if (isBootstrapSkillPath(args[0])) {
  14. existsCount += 1;
  15. }
  16. return originalExistsSync.apply(this, args);
  17. };
  18. fs.readFileSync = function (...args) {
  19. if (isBootstrapSkillPath(args[0])) {
  20. readCount += 1;
  21. }
  22. return originalReadFileSync.apply(this, args);
  23. };
  24. const mod = await import(pathToFileURL(pluginPath).href);
  25. const plugin = await mod.SuperpowersPlugin({ client: {}, directory: '.' });
  26. const transform = plugin['experimental.chat.messages.transform'];
  27. // Mapping constants are flavor-specific (#opencode-v2): V1 keeps the 1.18.x
  28. // tool names, V2 teaches the renamed tools. Assert both directly.
  29. const mappingFailures = assertMappingConstants(mod);
  30. const firstOutput = makeOutput(`${scenario} bootstrap first step`);
  31. await transform({}, firstOutput);
  32. const afterFirst = { existsCount, readCount };
  33. const secondOutput = makeOutput(`${scenario} bootstrap second step`);
  34. await transform({}, secondOutput);
  35. const afterSecond = { existsCount, readCount };
  36. // Exercise the V2 path (setup() + ctx.session.hook("context")) with a mock
  37. // ctx so the V2_MAPPING wiring is verified, not just the constant. Run after
  38. // the V1 count snapshots: setup() reads SKILL.md files during registration.
  39. const v2Result = await runV2ContextHook(mod);
  40. const result = {
  41. scenario,
  42. firstBootstrapParts: countBootstrapParts(firstOutput),
  43. secondBootstrapParts: countBootstrapParts(secondOutput),
  44. staleMentionMapping: bootstrapText(firstOutput).includes('@mention'),
  45. staleTaskMapping: bootstrapText(firstOutput).includes('`Task` tool with subagents'),
  46. mapsSubagentToTask: bootstrapText(firstOutput).includes('`task` with `subagent_type: "general"`'),
  47. mapsMutationToApplyPatch: bootstrapText(firstOutput).includes('`apply_patch`'),
  48. firstReadCount: afterFirst.readCount,
  49. secondReadCount: afterSecond.readCount,
  50. firstExistsCount: afterFirst.existsCount,
  51. secondExistsCount: afterSecond.existsCount,
  52. v2BootstrapParts: v2Result.bootstrapParts,
  53. mapsV2SubagentTool: v2Result.text.includes('`subagent` with `agent: "general"`'),
  54. mapsV2SessionIDContinuation: v2Result.text.includes('`sessionID` to continue a previous subagent'),
  55. mapsV2NoTodoTool: v2Result.text.includes('no todo tool'),
  56. mapsV2MutationToPatch: v2Result.text.includes('`patch` with `patchText`'),
  57. mapsV2Shell: v2Result.text.includes('`shell`'),
  58. staleV1ToolsInV2: v2Result.text.includes('`apply_patch`') || v2Result.text.includes('`todowrite`') || v2Result.text.includes('`subagent_type`'),
  59. };
  60. const failures = scenario === 'present'
  61. ? assertPresentBootstrap(result)
  62. : assertMissingBootstrap(result);
  63. if (scenario === 'present') {
  64. failures.push(...assertV2Bootstrap(result));
  65. }
  66. failures.push(...mappingFailures);
  67. if (failures.length > 0) {
  68. console.error(JSON.stringify(result, null, 2));
  69. for (const failure of failures) {
  70. console.error(`FAIL: ${failure}`);
  71. }
  72. process.exit(1);
  73. }
  74. console.log(JSON.stringify(result, null, 2));
  75. function isBootstrapSkillPath(filePath) {
  76. return String(filePath).replaceAll('\\', '/').includes('using-superpowers/SKILL.md');
  77. }
  78. function makeOutput(text) {
  79. return {
  80. messages: [{
  81. info: { role: 'user' },
  82. parts: [{ type: 'text', text }],
  83. }],
  84. };
  85. }
  86. function countBootstrapParts(output) {
  87. return output.messages[0].parts.filter(
  88. (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT')
  89. ).length;
  90. }
  91. function bootstrapText(output) {
  92. return output.messages[0].parts.find(
  93. (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT')
  94. )?.text || '';
  95. }
  96. function assertPresentBootstrap(result) {
  97. const failures = [];
  98. if (result.firstBootstrapParts !== 1) {
  99. failures.push(`expected first transform to inject one bootstrap part, got ${result.firstBootstrapParts}`);
  100. }
  101. if (result.secondBootstrapParts !== 1) {
  102. failures.push(`expected second transform to inject one bootstrap part, got ${result.secondBootstrapParts}`);
  103. }
  104. if (result.firstReadCount !== 1) {
  105. failures.push(`expected first transform to read SKILL.md once, got ${result.firstReadCount}`);
  106. }
  107. if (result.secondReadCount !== result.firstReadCount) {
  108. failures.push(`expected cached second transform to do no additional reads, got ${result.secondReadCount - result.firstReadCount}`);
  109. }
  110. if (result.secondExistsCount !== result.firstExistsCount) {
  111. failures.push(`expected cached second transform to do no additional exists checks, got ${result.secondExistsCount - result.firstExistsCount}`);
  112. }
  113. if (result.staleMentionMapping) {
  114. failures.push('expected OpenCode bootstrap not to teach @mention subagent syntax');
  115. }
  116. if (result.staleTaskMapping) {
  117. failures.push('expected OpenCode bootstrap not to teach stale Task-tool mapping');
  118. }
  119. if (!result.mapsSubagentToTask) {
  120. failures.push('expected OpenCode bootstrap to map general-purpose subagents to task with subagent_type');
  121. }
  122. if (!result.mapsMutationToApplyPatch) {
  123. failures.push('expected OpenCode bootstrap to map file mutation to apply_patch');
  124. }
  125. return failures;
  126. }
  127. function assertMissingBootstrap(result) {
  128. const failures = [];
  129. if (result.firstBootstrapParts !== 0) {
  130. failures.push(`expected no bootstrap when SKILL.md is missing, got ${result.firstBootstrapParts}`);
  131. }
  132. if (result.secondBootstrapParts !== 0) {
  133. failures.push(`expected no bootstrap on second missing-file transform, got ${result.secondBootstrapParts}`);
  134. }
  135. if (result.firstReadCount !== 0 || result.secondReadCount !== 0) {
  136. failures.push(`expected missing file path to avoid reads, got ${result.secondReadCount}`);
  137. }
  138. if (result.firstExistsCount < 1) {
  139. failures.push('expected first transform to check whether SKILL.md exists');
  140. }
  141. if (result.secondExistsCount !== result.firstExistsCount) {
  142. failures.push(`expected missing-file result to be cached, got ${result.secondExistsCount - result.firstExistsCount} extra exists checks`);
  143. }
  144. return failures;
  145. }
  146. function assertMappingConstants(mod) {
  147. const failures = [];
  148. if (typeof mod.V1_MAPPING !== 'string' || typeof mod.V2_MAPPING !== 'string') {
  149. failures.push('expected plugin to export V1_MAPPING and V2_MAPPING string constants');
  150. return failures;
  151. }
  152. for (const needle of ['`todowrite`', '`task` with `subagent_type: "general"`', '`apply_patch`', '`bash`']) {
  153. if (!mod.V1_MAPPING.includes(needle)) {
  154. failures.push(`expected V1_MAPPING to keep the 1.18.x tool name ${needle}`);
  155. }
  156. }
  157. for (const needle of [
  158. '`subagent` with `agent: "general"`',
  159. '`sessionID` to continue a previous subagent',
  160. 'no todo tool',
  161. '`write`',
  162. '`edit`',
  163. '`patch` with `patchText`',
  164. '`shell`',
  165. '`read`',
  166. '`grep`, `glob`',
  167. '`webfetch`',
  168. '`websearch`',
  169. ]) {
  170. if (!mod.V2_MAPPING.includes(needle)) {
  171. failures.push(`expected V2_MAPPING to teach the V2 tool ${needle}`);
  172. }
  173. }
  174. for (const stale of ['`todowrite`', '`task` with', '`apply_patch`', '`bash`']) {
  175. if (mod.V2_MAPPING.includes(stale)) {
  176. failures.push(`expected V2_MAPPING not to teach the V1-only tool name ${stale}`);
  177. }
  178. }
  179. return failures;
  180. }
  181. // Drive setup() with a mock V2 ctx and fire the captured "context" hook on a
  182. // top-level (parentID-less) session. Returns the injected-part count and the
  183. // injected bootstrap text ('' when nothing was injected).
  184. async function runV2ContextHook(mod) {
  185. let contextHook = null;
  186. const ctx = {
  187. skill: {
  188. transform: async (fn) => {
  189. fn({ add: () => {} });
  190. },
  191. },
  192. session: {
  193. hook: async (name, cb) => {
  194. if (name === 'context') contextHook = cb;
  195. },
  196. get: async ({ sessionID }) => ({ id: sessionID }), // top-level: no parentID
  197. },
  198. };
  199. try {
  200. await mod.default.setup(ctx);
  201. } catch (err) {
  202. console.error('[test] V2 setup() threw:', err);
  203. return { bootstrapParts: 0, text: '' };
  204. }
  205. if (typeof contextHook !== 'function') {
  206. return { bootstrapParts: 0, text: '' };
  207. }
  208. const event = {
  209. sessionID: 'sess-v2-top',
  210. messages: [{ role: 'user', content: [{ type: 'text', text: 'v2 bootstrap step' }] }],
  211. };
  212. await contextHook(event);
  213. const parts = event.messages[0].content.filter(
  214. (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT')
  215. );
  216. return { bootstrapParts: parts.length, text: parts[0]?.text || '' };
  217. }
  218. function assertV2Bootstrap(result) {
  219. const failures = [];
  220. if (result.v2BootstrapParts !== 1) {
  221. failures.push(`expected V2 context hook to inject one bootstrap part, got ${result.v2BootstrapParts}`);
  222. return failures;
  223. }
  224. if (!result.mapsV2SubagentTool) {
  225. failures.push('expected V2 bootstrap to map general-purpose subagents to subagent with agent');
  226. }
  227. if (!result.mapsV2SessionIDContinuation) {
  228. failures.push('expected V2 bootstrap to teach sessionID continuation for subagents');
  229. }
  230. if (!result.mapsV2NoTodoTool) {
  231. failures.push('expected V2 bootstrap to state that V2 has no todo tool');
  232. }
  233. if (!result.mapsV2MutationToPatch) {
  234. failures.push('expected V2 bootstrap to map file mutation to patch with patchText');
  235. }
  236. if (!result.mapsV2Shell) {
  237. failures.push('expected V2 bootstrap to map shell commands to the shell tool');
  238. }
  239. if (result.staleV1ToolsInV2) {
  240. failures.push('expected V2 bootstrap not to teach V1-only tool names (apply_patch/todowrite/subagent_type)');
  241. }
  242. return failures;
  243. }