verify-package-readme-model-experience.ts 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. /**
  2. * Doc-sync gate for package README Model Experience sections. It validates
  3. * audited package classifications, model/token/KV-cache fields, package-owned
  4. * text blocks, generated-catalog links, and final-section order. See the
  5. * [Model Experience Agent Note](../.agents/notes/implemented/process/2026-07-12-package-model-experience-contract.md).
  6. */
  7. import { existsSync, globSync, readFileSync } from 'node:fs'
  8. import { relative, resolve, sep } from 'node:path'
  9. import { markdownHeadingLines, markdownProseLines, type MarkdownProseLine } from './markdown.ts'
  10. const root = resolve(import.meta.dirname, '..')
  11. const HEADING = '## Model Experience'
  12. const LIMITATIONS_HEADING = '## Known Limitations and Deferred Work'
  13. const MODEL_VIEW_HEADING = '#### What the model sees'
  14. const TOKEN_EFFECT_HEADING = '#### Token effect'
  15. const KV_CACHE_EFFECT_HEADING = '#### KV Cache effect'
  16. const FIELD_HEADINGS = [MODEL_VIEW_HEADING, TOKEN_EFFECT_HEADING, KV_CACHE_EFFECT_HEADING] as const
  17. type SentenceKind = 'none' | 'indirect'
  18. interface SentenceContract {
  19. kind: SentenceKind
  20. reason: string
  21. }
  22. /**
  23. * Generic packages whose public contract is model-agnostic. Their READMEs omit
  24. * Model Experience entirely; the reason stays here as reviewable audit evidence
  25. * so an absent section cannot be mistaken for forgotten documentation.
  26. */
  27. const NO_MODEL_EXPERIENCE_SECTION: Readonly<Record<string, string>> = {
  28. 'packages/core/scope': 'The package is a model-agnostic registration and lifecycle primitive; model-facing consumers own any context selection.',
  29. 'packages/util/brand': 'The package is a type-only primitive erased at compile time.',
  30. 'packages/util/paths': 'The package only resolves harness-owned host paths; model-facing consumers own any rendered use.',
  31. 'packages/util/environment': 'The package only resolves host environment values; model-facing consumers own any rendered use.',
  32. }
  33. /**
  34. * Packages whose Model Experience is simple enough for one gated sentence plus
  35. * a KV-cache field. Every other package must carry canonical context-surface
  36. * blocks. A package moves on or off this list with its context behavior.
  37. */
  38. const SENTENCE_MODEL_EXPERIENCE: Readonly<Record<string, SentenceContract>> = {
  39. 'packages/attachment/attachment': { kind: 'indirect', reason: 'The storage seam delegates model request rendering to provider adapters.' },
  40. 'packages/attachment/attachment-local': { kind: 'indirect', reason: 'The local backend delegates model request rendering to provider adapters.' },
  41. 'packages/bash/bash': { kind: 'indirect', reason: 'The service interface delegates all model rendering to dsh-tool-bash.' },
  42. 'packages/bash/bash-env': { kind: 'indirect', reason: 'The env service surfaces managed DSH_* facts through the shell tools (dsh-tool-bash/dsh-tool-pwsh); it registers no prompt or schema of its own.' },
  43. 'packages/bash/bash-local': { kind: 'indirect', reason: 'The executor backend delegates model rendering to dsh-tool-bash.' },
  44. 'packages/bash/pwsh-local': { kind: 'indirect', reason: 'The executor backend delegates model rendering to dsh-tool-pwsh.' },
  45. 'packages/code-runtime/code-runtime': { kind: 'indirect', reason: 'The service interface delegates model rendering to Code Mode in dsh-tools.' },
  46. 'packages/core/agent-tool-mode': { kind: 'indirect', reason: 'The row only selects between the two projections dsh-tools owns; it registers no prompt, schema, or result of its own.' },
  47. 'packages/code-runtime/code-runtime-worker': { kind: 'indirect', reason: 'The worker backend delegates model rendering to Code Mode in dsh-tools.' },
  48. 'packages/client/ui-agent-preset': { kind: 'indirect', reason: 'Browser-side settings row; the preset it selects owns every model-facing effect.' },
  49. 'packages/core/agent-default-model': { kind: 'indirect', reason: 'The service supplies a ModelSelection; request assembly and adapters own the model-visible request.' },
  50. 'packages/preset/agent-presets': { kind: 'indirect', reason: 'The mount installs a preset\'s own plugins, which own every model-facing registration it makes visible.' },
  51. 'packages/typert/registry': { kind: 'none', reason: 'Runtime type registry; consumers (cordis_inspect, wire faces, gates) own any model-visible projection of registry contents.' },
  52. 'packages/typert/loader': { kind: 'none', reason: 'Loader integration only registers generated artifacts; consumers own any model-visible projection.' },
  53. 'packages/e2b/e2b': { kind: 'none', reason: 'The shared remote-runtime owner registers no model context; provider adapters and consumers own rendered effects.' },
  54. 'packages/client/hmr': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  55. 'packages/client/modules': { kind: 'none', reason: 'Browser-side module-loading kernel machinery; registers no model surface.' },
  56. 'packages/client/test-runtime': { kind: 'none', reason: 'Browser-side test infrastructure (jsdom bench); registers no model surface.' },
  57. 'packages/client/ui-slots': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  58. 'packages/client/ui-primitives': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  59. 'packages/client/web-react': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  60. 'packages/client/schema-form': { kind: 'none', reason: 'Browser-side form-rendering library; registers no model surface.' },
  61. 'packages/client/connection': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  62. 'packages/api/remotes': { kind: 'none', reason: 'The Remote BFF selects business methods and identity policy; selected services own any model-visible effect.' },
  63. 'packages/client/runtime': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  64. 'packages/client/ui-layout': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  65. 'packages/client/ui-sidebar': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  66. 'packages/client/ui-conversation': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  67. 'packages/client/ui-tool': { kind: 'none', reason: 'Browser-side Tool presentation layer; renders logged calls without changing model context.' },
  68. 'packages/client/ui-deliverables': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  69. 'packages/client/ui-task': { kind: 'none', reason: 'Browser-side read-only projection of ctx.tasks records; dsh-tool-tasks owns the model-facing surface.' },
  70. 'packages/client/ui-slash': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  71. 'packages/client/ui-command': { kind: 'indirect', reason: 'The dispatch paths trigger the host command.execute RPC; each command handler\'s host package owns any model-visible effect.' },
  72. 'packages/client/ui-model': { kind: 'indirect', reason: 'Selection routes session.selectModel; the Host snapshots the selection at the next prompt-assembly boundary and owns the model-visible effect.' },
  73. 'packages/client/ui-goal': { kind: 'indirect', reason: 'The strip verbs route goal.* mutations; the host GoalService owns the model-visible goal/change context message.' },
  74. 'packages/client/ui-permission': { kind: 'indirect', reason: 'The picker submits the host /permission command; the knob events it appends own the model-visible effect through the sandbox/approval consumers.' },
  75. 'packages/client/ui-plan': { kind: 'indirect', reason: 'The chip dispatches /plan off; dsh-plan-mode owns the model-visible policy, exit tool, and logged state.' },
  76. 'packages/client/ui-question': { kind: 'indirect', reason: 'The package mounts dsh-tool-ask-user; that tool owns the model-visible schema and answer rendering.' },
  77. 'packages/client/ui-trajectory': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  78. 'packages/client/ui-workspace': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  79. 'packages/client/ui-theme': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  80. 'packages/client/ui-settings': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  81. 'packages/client/ui-settings-general': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  82. 'packages/client/ui-models': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  83. 'packages/client/locale': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  84. 'packages/client/web': { kind: 'none', reason: 'Browser-side UI plugin layer; registers no model surface.' },
  85. 'packages/examples/agent-spine-demo': { kind: 'indirect', reason: 'The bundle only mounts model-facing child plugins.' },
  86. 'packages/fs/fs': { kind: 'indirect', reason: 'The service interface delegates model rendering to dsh-tool-fs.' },
  87. 'packages/e2b/fs-e2b': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-fs.' },
  88. 'packages/fs/fs-local': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-fs.' },
  89. 'packages/hooks/hook-protocol': { kind: 'indirect', reason: 'Only the hook bridge plugins render decoded hook output to a model.' },
  90. 'packages/host/apiproxy': { kind: 'none', reason: 'The wire contract and fetch carriers move already-composed messages and register no model surface.' },
  91. 'packages/host/directory-picker': { kind: 'none', reason: 'The GUI-host picking seam registers no model surface.' },
  92. 'packages/host/directory-picker-auto': { kind: 'none', reason: 'The GUI-host picking chooser only mounts a backend row; registers no model surface.' },
  93. 'packages/host/directory-picker-browse': { kind: 'none', reason: 'The GUI-host picking backend registers no model surface.' },
  94. 'packages/host/directory-picker-native': { kind: 'none', reason: 'The GUI-host picking backend registers no model surface.' },
  95. 'packages/host/webserver': { kind: 'none', reason: 'The HTTP carrier bridges browser and API handler and registers no model surface.' },
  96. 'packages/host/frontend-static': { kind: 'none', reason: 'The SPA dist server answers browser asset requests and registers no model surface.' },
  97. 'packages/bundle/base': { kind: 'indirect', reason: 'The bundle is a patch-list carrier; each inserted row\'s package owns its model surface.' },
  98. 'packages/bundle/headless': { kind: 'none', reason: 'The one-shot runner submits the task as an ordinary user message; prompts and tools belong to the composed base and headless bundles.' },
  99. 'packages/llm/llm': { kind: 'none', reason: 'The adapter registry forwards already-assembled requests unchanged.' },
  100. 'packages/llm/token-meter': { kind: 'indirect', reason: 'The measurement service leaves model-visible changes to its consumers.' },
  101. 'packages/lsp/lsp': { kind: 'indirect', reason: 'The provider registry delegates model rendering to dsh-tool-lsp.' },
  102. 'packages/lsp/lsp-local': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-lsp.' },
  103. 'packages/subprocess/subprocess': { kind: 'indirect', reason: 'The seam delegates all model rendering to consumer seams such as the bash executor family.' },
  104. 'packages/e2b/subprocess-e2b': { kind: 'indirect', reason: 'The remote spawn backend delegates model rendering to consumer seams such as the bash executor family.' },
  105. 'packages/subprocess/subprocess-local': { kind: 'indirect', reason: 'The spawn backend delegates model rendering to consumer seams such as the bash executor family.' },
  106. 'packages/sandbox/sandbox-local': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-bash-sandbox and dsh-tool-bash.' },
  107. 'packages/sandbox/sandbox-windows-acl': { kind: 'indirect', reason: 'The provider backend delegates model rendering to the bash/pwsh sandbox executors and their tools.' },
  108. 'packages/scaffold/create-sdk': { kind: 'indirect', reason: 'The initializer only writes project files; selected runtime plugins provide the generated project model surface.' },
  109. 'packages/scaffold/helper': { kind: 'none', reason: 'The project domain edits files and registers no live agent or model surface.' },
  110. 'packages/scaffold/scripts': { kind: 'indirect', reason: 'The launcher delegates model context to the loaded project plugin tree.' },
  111. 'packages/scaffold/client': { kind: 'none', reason: 'Client-process library; the model surface lives in the spawned runtime\'s composed plugins.' },
  112. 'packages/scaffold/protocol': { kind: 'none', reason: 'Client-facing wire library; the runtime plugins behind the serving entry own the model surface.' },
  113. 'packages/scaffold/telemetry': { kind: 'none', reason: 'The launcher-side reporter sends developer-cycle telemetry and registers no live agent or model surface.' },
  114. 'packages/session/session-projection': { kind: 'none', reason: 'The projection registry serves client-facing read models of already-logged session state and registers no model surface.' },
  115. 'packages/session/session-projection-cache': { kind: 'none', reason: 'The persisted cache accelerates host-side cold reads of projection state and registers no model surface.' },
  116. 'packages/session-query/session-query': { kind: 'none', reason: 'The trusted query service exposes cloned records only to callers and registers no model surface.' },
  117. 'packages/session-query/session-query-sqlite': { kind: 'none', reason: 'The search backend returns hits only to callers and registers no model surface.' },
  118. 'packages/settings/settings': { kind: 'indirect', reason: 'The seam stores and resolves user settings; consumer plugins own any model surface a value feeds.' },
  119. 'packages/settings/settings-local': { kind: 'indirect', reason: 'The file provider stores and publishes namespace sections; consumers of ctx.settings own any model surface.' },
  120. 'packages/credentials/credentials': { kind: 'indirect', reason: 'The seam resolves credential references; the consuming adapter owns every model surface a value authorizes.' },
  121. 'packages/credentials/credentials-local': { kind: 'indirect', reason: 'The file/environment provider stores credential values; consumers of ctx.credentials own any model surface.' },
  122. 'packages/util/atomic-write': { kind: 'none', reason: 'Pure filesystem write primitive; registers no model surface.' },
  123. 'packages/session/session-telemetry': { kind: 'none', reason: 'The seam observes the session stream and hands redacted copies outward; it registers no model surface.' },
  124. 'packages/session/session-telemetry-otel': { kind: 'none', reason: 'The backend forwards seam records into the OTel SDK pipeline and registers no model surface.' },
  125. 'packages/session/user-id': { kind: 'none', reason: 'The shared identifier appears only in telemetry metadata and a direct human command response; it registers no model surface.' },
  126. 'packages/skill/skill': { kind: 'indirect', reason: 'The provider registry delegates model rendering to dsh-tool-skill.' },
  127. 'packages/skill/skill-badge': { kind: 'indirect', reason: 'The bundled provider delegates model rendering to dsh-tool-skill.' },
  128. 'packages/skill/skill-local': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-skill.' },
  129. 'packages/spill/spill': { kind: 'indirect', reason: 'The storage seam delegates model rendering to spill consumers.' },
  130. 'packages/spill/spill-local': { kind: 'indirect', reason: 'The storage backend delegates model rendering to spill consumers.' },
  131. 'packages/support/acp-snapshot': { kind: 'none', reason: 'The test harness observes and normalizes transcripts without changing live requests.' },
  132. 'packages/support/agent-loop-testkit': { kind: 'none', reason: 'The test helper mounts services but neither drives nor modifies model requests.' },
  133. 'packages/support/invariants': { kind: 'none', reason: 'The observer validates requests but never rewrites their context.' },
  134. 'packages/support/loader-smoke': { kind: 'none', reason: 'The test harness submits an ordinary user task but delegates prompt and tool composition to the loaded tree.' },
  135. 'packages/support/llm-mock-server': { kind: 'none', reason: 'The test server substitutes provider wire behavior without invoking a real model.' },
  136. 'packages/support/llm-replay': { kind: 'none', reason: 'The keyless adapter invokes no provider model.' },
  137. 'packages/api/gateway': { kind: 'none', reason: 'Remote dispatch infrastructure; invoked business methods own any model-visible effect.' },
  138. 'packages/typert/type-meta': { kind: 'none', reason: 'Compiler-independent Remote protocol declarations; registers no model surface.' },
  139. 'packages/typert/generator': { kind: 'none', reason: 'The build-time generator runs outside any agent runtime and touches no model request.' },
  140. 'packages/tasks/tasks': { kind: 'indirect', reason: 'Producer and control-surface plugins own all model rendering over the task registry.' },
  141. 'packages/tasks/tasks-local': { kind: 'indirect', reason: 'The registry backend delegates model rendering to producer plugins and dsh-tool-tasks.' },
  142. 'packages/examples/acp-demo': { kind: 'indirect', reason: 'The app bundle delegates request composition to dsh-agent-spine-demo and dsh-acp.' },
  143. 'packages/boot/app-boot': { kind: 'indirect', reason: 'Only the loaded plugin tree contributes model context.' },
  144. 'packages/boot/cmdline': { kind: 'none', reason: 'Resolves the process command line before any session exists; configured rows own every model-visible consequence.' },
  145. 'packages/examples/jsonrpc-demo': { kind: 'indirect', reason: 'Only the externally configured plugin tree contributes model context.' },
  146. 'packages/interaction/permission': { kind: 'indirect', reason: 'The service writes mechanism events rendered by dsh-user-approval and dsh-tool-bash.' },
  147. 'packages/interaction/user-interaction': { kind: 'indirect', reason: 'Model-facing consumers render provider answers and seam errors.' },
  148. 'packages/util/timeout': { kind: 'indirect', reason: 'Only timeout consumers render timeout outcomes.' },
  149. 'packages/util/retention': { kind: 'indirect', reason: 'Only retention consumers render retained content and omission metadata.' },
  150. 'packages/util/native-command': { kind: 'none', reason: 'The host-side subprocess runner registers no model surface.' },
  151. 'packages/web/web': { kind: 'indirect', reason: 'The provider registry delegates model rendering to dsh-tool-web.' },
  152. 'packages/web/web-fetch-local': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-web.' },
  153. 'packages/web/web-search-exa': { kind: 'indirect', reason: 'The provider backend delegates model rendering to dsh-tool-web.' },
  154. 'packages/workflow/workflow': { kind: 'indirect', reason: 'The service delegates parent and child model rendering to its consumer and engine.' },
  155. }
  156. interface Failure {
  157. path: string
  158. message: string
  159. }
  160. type Line = MarkdownProseLine
  161. interface ContextSurface {
  162. heading: Line
  163. modelView: Line
  164. tokenEffect: Line
  165. kvCacheEffect: Line
  166. title: string
  167. modelViewVerbatimBlocks: number
  168. verbatimBlocks: number
  169. }
  170. interface ParsedField {
  171. value: Line
  172. verbatimBlocks: number
  173. }
  174. /** Validate H5-plus-markdown literals nested under one Model Experience field. */
  175. function validateNestedVerbatim(raw: readonly string[], fragments: Set<string>): { blocks: number; error?: string } {
  176. let cursor = 0
  177. while (raw[cursor]?.trim().length === 0) cursor += 1
  178. if (cursor === raw.length) return { blocks: 0 }
  179. let blocks = 0
  180. while (true) {
  181. while (raw[cursor]?.trim().length === 0) cursor += 1
  182. if (cursor === raw.length) break
  183. if (!/^##### \S/.test(raw[cursor] ?? '')) {
  184. return { blocks, error: 'content after a field paragraph must be a titled H5 verbatim block' }
  185. }
  186. const title = (raw[cursor] as string).slice('##### '.length)
  187. const fragment = headingFragment(title)
  188. if (fragment.length === 0) return { blocks, error: 'verbatim H5 title must be non-empty' }
  189. if (fragments.has(fragment)) {
  190. return { blocks, error: `verbatim H5 title ${JSON.stringify(title)} is duplicated within its context surface` }
  191. }
  192. fragments.add(fragment)
  193. cursor += 1
  194. while (raw[cursor]?.trim().length === 0) cursor += 1
  195. if (raw[cursor] !== '```markdown') {
  196. return { blocks, error: 'each nested verbatim H5 requires an exact ```markdown fence' }
  197. }
  198. cursor += 1
  199. const contentStart = cursor
  200. while (cursor < raw.length && raw[cursor] !== '```') cursor += 1
  201. if (cursor === raw.length) return { blocks, error: 'unterminated nested ```markdown fence' }
  202. if (cursor === contentStart) return { blocks, error: 'nested ```markdown fence must not be empty' }
  203. cursor += 1
  204. blocks += 1
  205. }
  206. return { blocks }
  207. }
  208. /** GitHub-style fragment for the simple ASCII nested titles allowed by these rules. */
  209. function headingFragment(title: string): string {
  210. return title.toLowerCase().replaceAll('`', '').replaceAll(/[^a-z0-9 _-]/g, '').trim().replaceAll(/\s+/g, '-')
  211. }
  212. /** A direct stable system-prompt contribution, as named by the README rules. */
  213. function isDirectSystemPromptSurface(title: string): boolean {
  214. return /\bsystem prompt\b/i.test(title)
  215. }
  216. /** Anchored generated-catalog links in one model-view field. */
  217. function toolCatalogLinkFragments(text: string): string[] {
  218. return [...text.matchAll(/\]\(\.\.\/\.\.\/\.\.\/docs\/tool-catalog\.md#([a-z0-9_-]+)\)/g)]
  219. .map(match => match[1] as string)
  220. }
  221. const toolCatalogFragments = new Set<string>()
  222. for (const line of readFileSync(resolve(root, 'docs/tool-catalog.md'), 'utf8').split('\n')) {
  223. const title = /^## (.+)$/.exec(line)?.[1]
  224. if (title !== undefined) toolCatalogFragments.add(headingFragment(title))
  225. }
  226. const failures: Failure[] = []
  227. const packageJsons = globSync('packages/*/*/package.json', { cwd: root }).map(path => path.split(sep).join('/')).sort()
  228. const scannedPackages = new Set(packageJsons.map(path => path.slice(0, -'/package.json'.length)))
  229. let structuredCount = 0
  230. let contextSurfaceCount = 0
  231. let omittedSectionCount = 0
  232. let explainedNoneCount = 0
  233. let indirectCount = 0
  234. let verbatimBlockCount = 0
  235. let systemPromptSurfaceCount = 0
  236. let toolSchemaSurfaceCount = 0
  237. let kvCacheEffectCount = 0
  238. for (const [pkg, reason] of Object.entries(NO_MODEL_EXPERIENCE_SECTION)) {
  239. if (!scannedPackages.has(pkg)) {
  240. failures.push({ path: `${pkg}/README.md`, message: 'no-section allowlist entry does not name a scanned package' })
  241. }
  242. if (reason.trim().length === 0) {
  243. failures.push({ path: `${pkg}/README.md`, message: 'no-section allowlist entry must retain its audit justification' })
  244. }
  245. if (SENTENCE_MODEL_EXPERIENCE[pkg] !== undefined) {
  246. failures.push({ path: `${pkg}/README.md`, message: 'package cannot appear in both Model Experience allowlists' })
  247. }
  248. }
  249. for (const [pkg, contract] of Object.entries(SENTENCE_MODEL_EXPERIENCE)) {
  250. if (!scannedPackages.has(pkg)) {
  251. failures.push({ path: `${pkg}/README.md`, message: 'sentence allowlist entry does not name a scanned package' })
  252. }
  253. if (contract.reason.trim().length === 0) {
  254. failures.push({ path: `${pkg}/README.md`, message: 'sentence allowlist entry must justify why structured context surfaces are unnecessary' })
  255. }
  256. }
  257. for (const packageJson of packageJsons) {
  258. const pkg = packageJson.slice(0, -'/package.json'.length)
  259. const readme = packageJson.replace(/package\.json$/, 'README.md')
  260. const abs = resolve(root, readme)
  261. if (!existsSync(abs)) {
  262. failures.push({ path: readme, message: 'missing package README' })
  263. continue
  264. }
  265. const text = readFileSync(abs, 'utf8')
  266. const rawLines = text.split('\n')
  267. const lines = markdownProseLines(text)
  268. const headings = markdownHeadingLines(text)
  269. const h2Headings = headings.filter(heading => heading.depth === 2)
  270. const modelExperienceHeadings = headings.filter(heading => heading.text
  271. .trim().replaceAll(/\s+/g, ' ').toLowerCase() === 'model experience')
  272. const modelHeadings = modelExperienceHeadings.filter(heading => heading.depth === 2 && heading.raw === HEADING)
  273. if (NO_MODEL_EXPERIENCE_SECTION[pkg] !== undefined) {
  274. if (modelExperienceHeadings.length !== 0) {
  275. for (const heading of modelExperienceHeadings) {
  276. failures.push({ path: readme, message: `line ${heading.index}: audited model-agnostic package must omit every Model Experience heading; found ${JSON.stringify(heading.raw)}` })
  277. }
  278. } else {
  279. omittedSectionCount += 1
  280. }
  281. continue
  282. }
  283. const nonCanonicalModelHeading = modelExperienceHeadings.find(heading => heading.depth !== 2 || heading.raw !== HEADING)
  284. if (nonCanonicalModelHeading !== undefined) {
  285. failures.push({ path: readme, message: `line ${nonCanonicalModelHeading.index}: non-canonical Model Experience heading ${JSON.stringify(nonCanonicalModelHeading.raw)}; use exactly ${JSON.stringify(HEADING)}` })
  286. continue
  287. }
  288. const modelHeading = modelHeadings.at(0)
  289. if (modelHeading === undefined) {
  290. failures.push({
  291. path: readme,
  292. message: `missing ${HEADING}`,
  293. })
  294. continue
  295. }
  296. if (modelHeadings.length !== 1) {
  297. failures.push({ path: readme, message: `contains ${modelHeadings.length} copies of ${HEADING}` })
  298. continue
  299. }
  300. const modelH2Index = h2Headings.indexOf(modelHeading)
  301. const limitationsH2Index = h2Headings.findIndex(heading => heading.depth === 2 && heading.raw === LIMITATIONS_HEADING)
  302. if (limitationsH2Index >= 0) {
  303. if (modelH2Index !== h2Headings.length - 2 || limitationsH2Index !== h2Headings.length - 1) {
  304. failures.push({
  305. path: readme,
  306. message: `${HEADING} and ${LIMITATIONS_HEADING} must be the final two H2 sections, in that order`,
  307. })
  308. continue
  309. }
  310. } else if (modelH2Index !== h2Headings.length - 1) {
  311. failures.push({ path: readme, message: `${HEADING} must be the final H2 when ${LIMITATIONS_HEADING} is absent` })
  312. continue
  313. }
  314. const modelHeadingAt = lines.findIndex(line => line.index === modelHeading.index)
  315. const body = lines.slice(modelHeadingAt + 1)
  316. const h2Lines = new Set(h2Headings.map(heading => heading.index))
  317. const nextH2 = body.findIndex(line => h2Lines.has(line.index))
  318. const section = nextH2 < 0 ? body : body.slice(0, nextH2)
  319. const nextH2Line = nextH2 < 0 ? rawLines.length + 1 : (body[nextH2] as Line).index
  320. const rawSection = rawLines.slice(modelHeading.index, nextH2Line - 1)
  321. const content = section.filter(line => line.raw.trim().length > 0)
  322. const sentenceContract = SENTENCE_MODEL_EXPERIENCE[pkg]
  323. if (sentenceContract !== undefined) {
  324. const pattern = sentenceContract.kind === 'none' ? /^None, as .+\.$/ : /^Indirectly, through .+\.$/
  325. const rawContent = rawSection.filter(line => line.trim().length > 0)
  326. const sentence = content[0]
  327. const kvCacheHeading = content[1]
  328. const kvCacheEffect = content[2]
  329. if (content.length !== 3 || rawContent.length !== 3 || !pattern.test(sentence?.raw ?? '')) {
  330. const prefix = sentenceContract.kind === 'none' ? 'None, as ' : 'Indirectly, through '
  331. failures.push({ path: readme, message: `must contain exactly one sentence beginning ${JSON.stringify(prefix)} and ending with a period, followed by ${KV_CACHE_EFFECT_HEADING} and one non-empty paragraph` })
  332. continue
  333. }
  334. if (kvCacheHeading?.raw !== KV_CACHE_EFFECT_HEADING
  335. || kvCacheEffect === undefined
  336. || /^#{1,6} /.test(kvCacheEffect.raw)
  337. || kvCacheEffect.raw.trim().length === 0) {
  338. failures.push({ path: readme, message: `line ${kvCacheHeading?.index ?? sentence?.index ?? modelHeading.index}: short Model Experience form requires exact ${KV_CACHE_EFFECT_HEADING} and one non-empty paragraph` })
  339. continue
  340. }
  341. if (sentence === undefined
  342. || sentence.index !== modelHeading.index + 2
  343. || kvCacheHeading.index !== sentence.index + 2
  344. || kvCacheEffect.index !== kvCacheHeading.index + 2) {
  345. failures.push({ path: readme, message: 'short Model Experience sentence, KV-cache H4, and paragraph require one blank line between each element' })
  346. continue
  347. }
  348. if (sentenceContract.kind === 'none') explainedNoneCount += 1
  349. else indirectCount += 1
  350. kvCacheEffectCount += 1
  351. continue
  352. }
  353. const shortSentence = content.find(line => line.raw === 'None.' || /^None, as |^Indirectly, through /.test(line.raw))
  354. if (shortSentence !== undefined) {
  355. failures.push({ path: readme, message: `line ${shortSentence.index}: short Model Experience form requires an audited entry in SENTENCE_MODEL_EXPERIENCE` })
  356. continue
  357. }
  358. const surfaceStarts = content
  359. .map((line, index) => ({ line, index }))
  360. .filter(entry => /^### \S/.test(entry.line.raw))
  361. if (surfaceStarts.length === 0 || surfaceStarts[0]?.index !== 0) {
  362. failures.push({ path: readme, message: 'must contain one or more complete context-surface blocks' })
  363. continue
  364. }
  365. const surfaces: ContextSurface[] = []
  366. const surfaceFragments = new Set<string>()
  367. let surfaceError = false
  368. for (let surfaceIndex = 0; surfaceIndex < surfaceStarts.length; surfaceIndex += 1) {
  369. const start = surfaceStarts[surfaceIndex] as { line: Line; index: number }
  370. const end = surfaceStarts[surfaceIndex + 1]?.index ?? content.length
  371. const entries = content.slice(start.index, end)
  372. const heading = entries[0] as Line
  373. const title = heading.raw.slice('### '.length)
  374. const fragment = headingFragment(title)
  375. if (fragment.length === 0) {
  376. failures.push({ path: readme, message: `line ${heading.index}: each context surface requires a non-empty H3 heading` })
  377. surfaceError = true
  378. break
  379. }
  380. if (surfaceFragments.has(fragment)) {
  381. failures.push({ path: readme, message: `line ${heading.index}: duplicate context-surface link fragment ${JSON.stringify(fragment)}` })
  382. surfaceError = true
  383. break
  384. }
  385. const fieldStarts = entries
  386. .map((line, index) => ({ line, index }))
  387. .filter(entry => /^#### \S/.test(entry.line.raw))
  388. if (fieldStarts.length !== FIELD_HEADINGS.length || fieldStarts[0]?.index !== 1) {
  389. failures.push({ path: readme, message: `line ${heading.index}: context surface requires exactly three ordered H4 fields: ${FIELD_HEADINGS.join(', ')}` })
  390. surfaceError = true
  391. break
  392. }
  393. if ((surfaceIndex === 0 && heading.index !== modelHeading.index + 2)
  394. || rawLines[heading.index - 2]?.trim().length !== 0
  395. || fieldStarts[0].line.index !== heading.index + 2) {
  396. failures.push({ path: readme, message: `line ${heading.index}: context-surface heading and first field require one blank line between them` })
  397. surfaceError = true
  398. break
  399. }
  400. const parsedFields: ParsedField[] = []
  401. const verbatimFragments = new Set<string>()
  402. for (let fieldIndex = 0; fieldIndex < FIELD_HEADINGS.length; fieldIndex += 1) {
  403. const fieldStart = fieldStarts[fieldIndex] as { line: Line; index: number }
  404. const expectedHeading = FIELD_HEADINGS[fieldIndex] as string
  405. if (fieldStart.line.raw !== expectedHeading) {
  406. failures.push({ path: readme, message: `line ${fieldStart.line.index}: expected exact field heading ${JSON.stringify(expectedHeading)}, found ${JSON.stringify(fieldStart.line.raw)}` })
  407. surfaceError = true
  408. break
  409. }
  410. const fieldEnd = fieldStarts[fieldIndex + 1]?.index ?? entries.length
  411. const fieldEntries = entries.slice(fieldStart.index, fieldEnd)
  412. const value = fieldEntries[1]
  413. if (value === undefined || /^#{1,6} /.test(value.raw) || value.raw.trim().length === 0) {
  414. failures.push({ path: readme, message: `line ${fieldStart.line.index}: ${expectedHeading} requires one non-empty paragraph` })
  415. surfaceError = true
  416. break
  417. }
  418. if (value.index !== fieldStart.line.index + 2) {
  419. failures.push({ path: readme, message: `line ${fieldStart.line.index}: ${expectedHeading} and its paragraph require one blank line between them` })
  420. surfaceError = true
  421. break
  422. }
  423. const unexpected = fieldEntries.slice(2).find(line => !/^##### \S/.test(line.raw))
  424. if (unexpected !== undefined) {
  425. failures.push({ path: readme, message: `line ${unexpected.index}: content after ${expectedHeading} paragraph must be a titled H5 plus \`markdown\` fence owned by that field` })
  426. surfaceError = true
  427. break
  428. }
  429. const nextHeadingLine = fieldStarts[fieldIndex + 1]?.line.index
  430. ?? surfaceStarts[surfaceIndex + 1]?.line.index
  431. ?? nextH2Line
  432. if (rawLines[nextHeadingLine - 2]?.trim().length !== 0) {
  433. failures.push({ path: readme, message: `line ${nextHeadingLine}: Model Experience headings require a preceding blank line` })
  434. surfaceError = true
  435. break
  436. }
  437. const verbatim = validateNestedVerbatim(rawLines.slice(value.index, nextHeadingLine - 1), verbatimFragments)
  438. if (verbatim.error !== undefined) {
  439. failures.push({ path: readme, message: `line ${value.index}: ${verbatim.error}` })
  440. surfaceError = true
  441. break
  442. }
  443. if (fieldEntries.length - 2 !== verbatim.blocks) {
  444. failures.push({ path: readme, message: `line ${value.index}: every nested H5 must own exactly one \`markdown\` fence` })
  445. surfaceError = true
  446. break
  447. }
  448. parsedFields.push({ value, verbatimBlocks: verbatim.blocks })
  449. }
  450. if (surfaceError) break
  451. const modelViewField = parsedFields[0] as ParsedField
  452. const tokenEffectField = parsedFields[1] as ParsedField
  453. const kvCacheEffectField = parsedFields[2] as ParsedField
  454. const modelView = modelViewField.value
  455. const tokenEffect = tokenEffectField.value
  456. const kvCacheEffect = kvCacheEffectField.value
  457. if (/\]\(#[^)]+\)/.test(modelView.raw) || /\]\(#[^)]+\)/.test(tokenEffect.raw) || /\]\(#[^)]+\)/.test(kvCacheEffect.raw)) {
  458. failures.push({ path: readme, message: `line ${heading.index}: Model Experience fields must not link between local subsections; nest the H5 in its owning H4 field` })
  459. surfaceError = true
  460. break
  461. }
  462. surfaceFragments.add(fragment)
  463. surfaces.push({
  464. heading,
  465. modelView,
  466. tokenEffect,
  467. kvCacheEffect,
  468. title,
  469. modelViewVerbatimBlocks: modelViewField.verbatimBlocks,
  470. verbatimBlocks: parsedFields.reduce((total, field) => total + field.verbatimBlocks, 0),
  471. })
  472. }
  473. if (surfaceError) continue
  474. const promptWithoutVerbatim = surfaces.find(surface => isDirectSystemPromptSurface(surface.title)
  475. && surface.modelViewVerbatimBlocks === 0)
  476. if (promptWithoutVerbatim !== undefined) {
  477. failures.push({ path: readme, message: `line ${promptWithoutVerbatim.heading.index}: system-prompt surface must contain a titled H5 plus verbatim \`markdown\` block under ${MODEL_VIEW_HEADING}` })
  478. continue
  479. }
  480. const hasConcreteLiteral = surfaces.some(surface => surface.verbatimBlocks > 0
  481. || surface.modelView.raw.includes('`')
  482. || surface.tokenEffect.raw.includes('`')
  483. || toolCatalogLinkFragments(surface.modelView.raw).length > 0)
  484. if (!hasConcreteLiteral) {
  485. failures.push({ path: readme, message: 'structured Model Experience must ground at least one surface with inline code, a nested `markdown` block, or an anchored tool-catalog link' })
  486. continue
  487. }
  488. let catalogError = false
  489. for (const surface of surfaces) {
  490. if (!/\bschemas?\b/i.test(surface.title)) continue
  491. const fragments = toolCatalogLinkFragments(surface.modelView.raw)
  492. if (fragments.length === 0) {
  493. failures.push({ path: readme, message: `line ${surface.heading.index}: tool-schema surface must link an anchored section of ../../../docs/tool-catalog.md` })
  494. catalogError = true
  495. break
  496. }
  497. const invalid = fragments.find(fragment => !toolCatalogFragments.has(fragment))
  498. if (invalid !== undefined) {
  499. failures.push({ path: readme, message: `line ${surface.modelView.index}: tool-catalog link fragment ${JSON.stringify(invalid)} does not name an H2 section` })
  500. catalogError = true
  501. break
  502. }
  503. }
  504. if (catalogError) continue
  505. verbatimBlockCount += surfaces.reduce((total, surface) => total + surface.verbatimBlocks, 0)
  506. contextSurfaceCount += surfaces.length
  507. systemPromptSurfaceCount += surfaces.filter(surface => isDirectSystemPromptSurface(surface.title)).length
  508. toolSchemaSurfaceCount += surfaces.filter(surface => /\bschemas?\b/i.test(surface.title)).length
  509. kvCacheEffectCount += surfaces.length
  510. structuredCount += 1
  511. }
  512. if (failures.length === 0) {
  513. console.log(`verify-package-readme-model-experience: ${packageJsons.length} README(s) checked (${omittedSectionCount} audited omissions, ${structuredCount} structured, ${contextSurfaceCount} context surfaces, ${kvCacheEffectCount} KV-cache fields, ${systemPromptSurfaceCount} fenced system-prompt surfaces, ${toolSchemaSurfaceCount} catalog-linked tool-schema surfaces, ${explainedNoneCount} explained none, ${indirectCount} indirect, ${verbatimBlockCount} verbatim markdown blocks), all conform.`)
  514. process.exit(0)
  515. }
  516. console.error('verify-package-readme-model-experience failed:')
  517. for (const failure of failures) {
  518. console.error(` ${relative(root, resolve(root, failure.path))}: ${failure.message}`)
  519. }
  520. process.exit(1)