token-meter.spec.ts 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId } from '@deepseek-ai/dsh-llm'
  4. import type { ContentBlock, Message, TokenUsage } from '@deepseek-ai/dsh-llm'
  5. import SessionStore, { Session, SessionId, canonicalHeader } from '@deepseek-ai/dsh-session'
  6. import type { EpochHeader, SessionEvent } from '@deepseek-ai/dsh-session'
  7. import TokenMeterService from '@deepseek-ai/dsh-token-meter'
  8. import type { TokenMeasurement, TokenMeterConfig } from '@deepseek-ai/dsh-token-meter'
  9. function header(model: string, extras: Omit<EpochHeader, 'config'> = {}): EpochHeader {
  10. return canonicalHeader({ config: { provider: 'mock', model }, ...extras })
  11. }
  12. function textMessage(text: string, role: Message['role'] = 'user'): Message {
  13. return { role, content: [{ type: 'text', text }] }
  14. }
  15. function appendHeader(session: Session, value: EpochHeader): void {
  16. session.append('request/header', { header: value, reason: 'initial' })
  17. }
  18. /** Inject malformed persisted history after the live append boundary for defensive replay tests. */
  19. function appendUnchecked(session: Session, event: SessionEvent): void {
  20. const log = (session as unknown as { log: SessionEvent[] }).log
  21. log.push(event)
  22. }
  23. interface SuccessfulCallOptions {
  24. turn?: number
  25. step?: number
  26. providerText?: string
  27. durableText?: string
  28. usage?: TokenUsage
  29. provenance?: 'exact' | 'empty' | 'absent'
  30. }
  31. function appendSuccessfulCall(
  32. session: Session,
  33. value: EpochHeader,
  34. options: SuccessfulCallOptions = {},
  35. ): void {
  36. const turn = options.turn ?? 1
  37. const step = options.step ?? 1
  38. const providerText = options.providerText ?? 'provider answer'
  39. const durableText = options.durableText ?? providerText
  40. const provenance = options.provenance ?? 'exact'
  41. session.append('step/start', { turn, step })
  42. appendHeader(session, value)
  43. const sources: number[] = []
  44. if (provenance === 'exact') {
  45. const chunks = [
  46. { type: 'block-start' as const, index: 0, blockType: 'text' as const },
  47. { type: 'text-delta' as const, index: 0, text: providerText },
  48. { type: 'block-end' as const, index: 0, block: { type: 'text' as const, text: providerText } },
  49. ...options.usage === undefined ? [] : [{ type: 'usage' as const, usage: options.usage }],
  50. { type: 'finish' as const, reason: { kind: 'stop' as const } },
  51. ]
  52. for (const chunk of chunks) {
  53. sources.push(session.append('assistant/chunk', { turn, step, chunk }).seq)
  54. }
  55. }
  56. const intent = provenance === 'absent'
  57. ? { surfaceOp: 'append' as const }
  58. : { surfaceOp: 'append' as const, sourceEventSeqs: provenance === 'empty' ? [] : sources }
  59. session.append('assistant/message', {
  60. provenance: {
  61. provider: value.config.provider,
  62. model: value.config.model,
  63. },
  64. turn,
  65. step,
  66. content: durableText.length === 0 ? [] : [{ type: 'text', text: durableText }],
  67. ...options.usage === undefined ? {} : { usage: options.usage },
  68. }, intent)
  69. session.append('step/end', { turn, step })
  70. }
  71. function meter(config: TokenMeterConfig = {}): TokenMeterService {
  72. return new TokenMeterService(new Context(), config)
  73. }
  74. function expectSurfaceTotal(measurement: TokenMeasurement): void {
  75. expect(measurement.nodes.reduce((total, node) => total + node.tokens, 0))
  76. .toBe(measurement.surfaceTokens)
  77. }
  78. describe('TokenMeterService configuration and registration', () => {
  79. it('provides one zero-config context window', () => {
  80. const service = meter()
  81. expect(service.contextWindow).toBe(128_000)
  82. })
  83. it('accepts one service-wide context-window override', () => {
  84. expect(meter({ contextWindow: 32_000 }).contextWindow).toBe(32_000)
  85. })
  86. it.each(['models', 'contextWidow'])('rejects unknown top-level config key %s', (key) => {
  87. expect(() => meter({ [key]: {} }))
  88. .toThrow(`TokenMeterConfig: unknown key "${key}"`)
  89. })
  90. it.each([
  91. { contextWindow: 0 },
  92. { contextWindow: -1 },
  93. { contextWindow: 1.5 },
  94. { contextWindow: Number.NaN },
  95. { contextWindow: null },
  96. ] as unknown as TokenMeterConfig[])('rejects invalid context capacity %#', (config) => {
  97. expect(() => meter(config)).toThrow(/contextWindow .* positive integer/)
  98. })
  99. it('registers and unregisters ctx.tokenMeter with its plugin fiber', async () => {
  100. const ctx = new Context()
  101. await ctx.plugin(SessionStore)
  102. const fiber = await ctx.plugin(TokenMeterService)
  103. expect(ctx.get('tokenMeter')).toBeInstanceOf(TokenMeterService)
  104. await fiber.dispose()
  105. expect(ctx.get('tokenMeter')).toBeUndefined()
  106. })
  107. })
  108. describe('TokenMeterService pricing', () => {
  109. it('prices every built-in content shape and merge-extended blocks with one fixed heuristic', () => {
  110. const service = meter({ contextWindow: 100 })
  111. const blocks: ContentBlock[] = [
  112. { type: 'text', text: 'abcd' },
  113. { type: 'reasoning', text: 'ab' },
  114. { type: 'tool-call', id: CallId('c'), name: 'read', arguments: '{"x":1}' },
  115. {
  116. type: 'tool-result',
  117. toolCallId: CallId('c'),
  118. content: [{ type: 'text', text: 'xy' }],
  119. isError: false,
  120. },
  121. { type: 'future-block', payload: 'abcd' } as unknown as ContentBlock,
  122. ]
  123. const estimated = service.estimateMessage({ role: 'assistant', content: blocks })
  124. expect(estimated).toBeGreaterThan(30)
  125. expect(service.estimateMessage(textMessage('abcd'))).toBe(9)
  126. })
  127. it('returns a detached deeply immutable empty measurement', () => {
  128. const service = meter()
  129. const session = new Session(SessionId('empty'))
  130. const result = service.measure(session)
  131. expect(result).toEqual({
  132. logRevision: 0,
  133. baseline: { kind: 'none', tokens: 0 },
  134. surfaceDeltaTokens: 0,
  135. totalTokens: 0,
  136. surfaceTokens: 0,
  137. nodes: [],
  138. })
  139. expect(Object.isFrozen(result)).toBe(true)
  140. expect(Object.isFrozen(result.baseline)).toBe(true)
  141. expect(Object.isFrozen(result.nodes)).toBe(true)
  142. expectSurfaceTotal(result)
  143. expect(() => {
  144. ;(result as { totalTokens: number }).totalTokens = 1
  145. }).toThrow(TypeError)
  146. })
  147. it('keeps an earlier unified snapshot detached from later replay', () => {
  148. const service = meter()
  149. const session = new Session(SessionId('detached'))
  150. session.append('user/message', {
  151. content: [{ type: 'text', text: 'first' }],
  152. source: { kind: 'user' },
  153. }, { surfaceOp: 'append' })
  154. const snapshot = service.measure(session)
  155. const snapshotCopy = structuredClone(snapshot)
  156. expect(Object.isFrozen(snapshot.nodes)).toBe(true)
  157. expect(Object.isFrozen(snapshot.nodes[0])).toBe(true)
  158. expectSurfaceTotal(snapshot)
  159. expect(() => {
  160. ;(snapshot.nodes as Array<{ seq: number; tokens: number }>).push({ seq: 99, tokens: 1 })
  161. }).toThrow(TypeError)
  162. expect(() => {
  163. ;(snapshot.nodes[0] as { seq: number; tokens: number }).tokens = 1
  164. }).toThrow(TypeError)
  165. session.append('user/message', {
  166. content: [{ type: 'text', text: 'second' }],
  167. source: { kind: 'user' },
  168. }, { surfaceOp: 'append' })
  169. const advanced = service.measure(session)
  170. expect(advanced.logRevision).toBe(2)
  171. expect(advanced.nodes).toHaveLength(2)
  172. expectSurfaceTotal(advanced)
  173. expect(snapshot).toEqual(snapshotCopy)
  174. expect(snapshot.logRevision).toBe(1)
  175. expect(snapshot.nodes).toHaveLength(1)
  176. })
  177. it('prices header, prefix, tools, and surface when no reusable usage exists', () => {
  178. const service = meter()
  179. const session = new Session(SessionId('heuristic'))
  180. session.append('user/message', {
  181. content: [{ type: 'text', text: 'question' }],
  182. source: { kind: 'user' },
  183. }, { surfaceOp: 'append' })
  184. appendHeader(session, header('deepseek-v4-flash', {
  185. system: 'system',
  186. messagePrefix: [textMessage('prefix')],
  187. tools: [{ name: 'read', description: 'read', parameters: { type: 'object' } }],
  188. }))
  189. const result = service.measure(session)
  190. expect(result.baseline.kind).toBe('estimated')
  191. expect(result.totalTokens).toBeGreaterThan(result.surfaceTokens)
  192. expect(result.logRevision).toBe(session.events.length)
  193. expectSurfaceTotal(result)
  194. })
  195. it('keeps request-header overrides out of the returned surface', () => {
  196. const service = meter()
  197. const session = new Session(SessionId('override-surface'))
  198. session.append('user/message', {
  199. content: [{ type: 'text', text: 'question' }],
  200. source: { kind: 'user' },
  201. }, { surfaceOp: 'append' })
  202. const logged = service.measure(session)
  203. const overridden = service.measure(session, header('another-model', {
  204. system: 'large override '.repeat(100),
  205. }))
  206. expect(overridden.totalTokens).toBeGreaterThan(logged.totalTokens)
  207. expect(overridden.surfaceTokens).toBe(logged.surfaceTokens)
  208. expect(overridden.nodes).toEqual(logged.nodes)
  209. expectSurfaceTotal(overridden)
  210. })
  211. })
  212. describe('replay anchors and surface folds', () => {
  213. const USAGE: TokenUsage = {
  214. inputTokens: 20,
  215. cacheReadTokens: 3,
  216. cacheWriteTokens: 4,
  217. outputTokens: 7,
  218. reasoningTokens: 6,
  219. }
  220. it('uses disjoint provider usage and signed durable-output rewrites', () => {
  221. const service = meter()
  222. const session = new Session(SessionId('usage'))
  223. session.append('user/message', {
  224. content: [{ type: 'text', text: 'before' }],
  225. source: { kind: 'user' },
  226. }, { surfaceOp: 'append' })
  227. appendSuccessfulCall(session, header('deepseek-v4-flash'), {
  228. providerText: 'short',
  229. durableText: 'a much longer rewritten durable assistant answer',
  230. usage: USAGE,
  231. })
  232. const result = service.measure(session)
  233. expect(result.baseline).toMatchObject({ kind: 'usage', tokens: 34, usage: USAGE })
  234. expect(result.surfaceDeltaTokens).toBeGreaterThan(0)
  235. expect(result.totalTokens).toBe(34 + result.surfaceDeltaTokens)
  236. expect(() => {
  237. ;((result.baseline as { usage: { inputTokens: number } }).usage.inputTokens) = 1
  238. }).toThrow(TypeError)
  239. })
  240. it('selects a heuristic anchor when provider usage would undercut its scale', () => {
  241. const service = meter()
  242. const session = new Session(SessionId('low-usage-anchor'))
  243. const system = 'system context'
  244. const requestHeader = header('deepseek-v4-flash', { system })
  245. appendSuccessfulCall(session, requestHeader, {
  246. providerText: 'abcd'.repeat(512),
  247. usage: { inputTokens: 20, outputTokens: 7 },
  248. })
  249. const anchored = service.measure(session)
  250. expect(anchored.baseline.kind).toBe('estimated')
  251. const assistant = anchored.nodes[0]!.seq
  252. session.append('user/message', {
  253. content: [{ type: 'text', text: 'short' }],
  254. source: { kind: 'plugin', plugin: 'test' },
  255. }, {
  256. surfaceOp: { op: 'replace', start: assistant, end: assistant },
  257. sourceEventSeqs: [assistant],
  258. })
  259. const shrunken = service.measure(session)
  260. expect(27 + shrunken.surfaceDeltaTokens).toBeLessThan(0)
  261. expect(shrunken.totalTokens).toBeGreaterThan(0)
  262. expect(shrunken.totalTokens).toBe(service.measure(
  263. session,
  264. header('different-model', { system }),
  265. ).totalTokens)
  266. })
  267. it('uses an estimated anchor when provider usage is absent', () => {
  268. const service = meter()
  269. const session = new Session(SessionId('missing-usage'))
  270. appendSuccessfulCall(session, header('deepseek-v4-flash', { system: 's' }), {
  271. providerText: 'provider',
  272. durableText: 'rewritten',
  273. })
  274. const anchored = service.measure(session)
  275. expect(anchored.baseline.kind).toBe('estimated')
  276. expect(anchored.surfaceDeltaTokens).toBe(0)
  277. session.append('user/message', {
  278. content: [{ type: 'text', text: 'later' }],
  279. source: { kind: 'user' },
  280. }, { surfaceOp: 'append' })
  281. const advanced = service.measure(session)
  282. expect(advanced.surfaceDeltaTokens).toBeGreaterThan(0)
  283. })
  284. it('distinguishes explicit empty provenance from absent legacy provenance', () => {
  285. const explicit = new Session(SessionId('explicit-empty'))
  286. const legacy = new Session(SessionId('legacy-absent'))
  287. appendSuccessfulCall(explicit, header('deepseek-v4-flash'), {
  288. durableText: 'listener injected text',
  289. providerText: '',
  290. usage: USAGE,
  291. provenance: 'empty',
  292. })
  293. appendSuccessfulCall(legacy, header('deepseek-v4-flash'), {
  294. durableText: 'listener injected text',
  295. providerText: '',
  296. usage: USAGE,
  297. provenance: 'absent',
  298. })
  299. const service = meter()
  300. expect(service.measure(explicit).surfaceDeltaTokens).toBeGreaterThan(0)
  301. expect(service.measure(legacy).surfaceDeltaTokens).toBe(0)
  302. })
  303. it('keeps only the latest successful request anchor across model switches', () => {
  304. const service = meter({ contextWindow: 1_000 })
  305. const session = new Session(SessionId('switch'))
  306. const alphaHeader = header('alpha', { system: 'same envelope' })
  307. appendSuccessfulCall(session, alphaHeader, { usage: USAGE, providerText: 'alpha' })
  308. expect(service.measure(session).baseline).toMatchObject({ kind: 'usage', tokens: 34 })
  309. appendSuccessfulCall(session, header('beta'), {
  310. turn: 1,
  311. step: 2,
  312. usage: { inputTokens: 100, outputTokens: 50 },
  313. providerText: 'beta response',
  314. })
  315. expect(service.measure(session).baseline).toMatchObject({ kind: 'usage', tokens: 150 })
  316. appendHeader(session, alphaHeader)
  317. const switchedBack = service.measure(session)
  318. expect(switchedBack.baseline.kind).toBe('estimated')
  319. expect(switchedBack.surfaceDeltaTokens).toBe(0)
  320. })
  321. it('invalidates usage for any canonical envelope change or explicit override', () => {
  322. const service = meter()
  323. const session = new Session(SessionId('envelope'))
  324. const anchoredHeader = header('deepseek-v4-flash', { system: 'one' })
  325. appendSuccessfulCall(session, anchoredHeader, { usage: USAGE })
  326. expect(service.measure(session, { ...anchoredHeader, tools: [] }).baseline.kind).toBe('usage')
  327. expect(service.measure(session, header('deepseek-v4-flash', { system: 'two' })).baseline.kind)
  328. .toBe('estimated')
  329. expect(service.measure(session, header('deepseek-v4-pro', { system: 'one' })).baseline.kind)
  330. .toBe('estimated')
  331. expect(service.measure(session, {
  332. ...anchoredHeader,
  333. config: { ...anchoredHeader.config, temperature: 0.2 },
  334. }).baseline.kind).toBe('estimated')
  335. expect(service.measure(session, {
  336. ...anchoredHeader,
  337. messagePrefix: [textMessage('prefix')],
  338. }).baseline.kind).toBe('estimated')
  339. expect(service.measure(session, {
  340. ...anchoredHeader,
  341. tools: [{ name: 'read', description: 'read', parameters: { type: 'object' } }],
  342. }).baseline.kind).toBe('estimated')
  343. })
  344. it('folds the latest full header snapshot into the effective envelope', () => {
  345. const session = new Session(SessionId('header-snapshot'))
  346. appendHeader(session, header('deepseek-v4-flash'))
  347. session.append('request/header', {
  348. header: header('deepseek-v4-pro'),
  349. reason: 'change',
  350. })
  351. const result = meter().measure(session)
  352. expect(result.baseline.kind).toBe('estimated')
  353. expect(result.logRevision).toBe(2)
  354. })
  355. it('replays seeded append and replace operations with signed deltas', () => {
  356. const service = meter()
  357. const original = new Session(SessionId('surface-original'))
  358. appendSuccessfulCall(original, header('deepseek-v4-flash'), {
  359. usage: USAGE,
  360. providerText: 'long provider answer '.repeat(100),
  361. })
  362. original.append('user/message', {
  363. content: [{ type: 'text', text: 'new tail' }],
  364. source: { kind: 'user' },
  365. }, { surfaceOp: 'append' })
  366. const seeded = new Session(SessionId('surface-seeded'), original.events)
  367. const before = service.measure(seeded)
  368. expect(before.nodes).toHaveLength(2)
  369. expect(before.surfaceDeltaTokens).toBeGreaterThan(0)
  370. expectSurfaceTotal(before)
  371. const first = seeded.surface.nodes[0]!
  372. seeded.append('user/message', {
  373. content: [{ type: 'text', text: 'replacement' }],
  374. source: { kind: 'plugin', plugin: 'test' },
  375. }, { surfaceOp: { op: 'replace', start: first, end: first }, sourceEventSeqs: [first] })
  376. const after = service.measure(seeded)
  377. expect(after.nodes).toHaveLength(2)
  378. expect(after.nodes[0]!.seq).toBe(seeded.events.length - 1)
  379. expect(after.logRevision).toBe(seeded.events.length)
  380. expect(Object.isFrozen(after.nodes)).toBe(true)
  381. expect(Object.isFrozen(after.nodes[0])).toBe(true)
  382. expect(after.surfaceDeltaTokens).toBeLessThan(0)
  383. expectSurfaceTotal(after)
  384. expect(before.nodes).toHaveLength(2)
  385. expect(before.logRevision).toBe(original.events.length)
  386. expect(before.surfaceDeltaTokens).toBeGreaterThan(0)
  387. })
  388. it('prices an empty assistant surface anchor as zero', () => {
  389. const session = new Session(SessionId('empty-assistant'))
  390. appendSuccessfulCall(session, header('deepseek-v4-flash'), {
  391. providerText: '',
  392. durableText: '',
  393. provenance: 'empty',
  394. })
  395. const measurement = meter().measure(session)
  396. const assistant = session.events.find(event => event.type === 'assistant/message')!
  397. expect(measurement.nodes).toEqual([{ seq: assistant.seq, tokens: 0 }])
  398. expect(measurement.surfaceTokens).toBe(0)
  399. expectSurfaceTotal(measurement)
  400. })
  401. })
  402. describe('malformed replay and listener lifecycle', () => {
  403. function expectRepeatedFailure(service: TokenMeterService, session: Session, pattern: RegExp): void {
  404. expect(() => service.measure(session)).toThrow(pattern)
  405. expect(() => service.measure(session)).toThrow(pattern)
  406. }
  407. it('rejects an assistant without its step boundary transactionally', () => {
  408. const session = new Session(SessionId('bad-step'))
  409. appendHeader(session, header('deepseek-v4-flash'))
  410. session.append('assistant/message', {
  411. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  412. turn: 1,
  413. step: 1,
  414. content: [{ type: 'text', text: 'bad' }],
  415. }, { surfaceOp: 'append', sourceEventSeqs: [] })
  416. expectRepeatedFailure(meter(), session, /no matching step\/start/)
  417. })
  418. it('clears completed step boundaries and rejects overlapping or late step events', () => {
  419. const overlapping = new Session(SessionId('overlapping-step'))
  420. overlapping.append('step/start', { turn: 1, step: 1 })
  421. overlapping.append('step/start', { turn: 1, step: 2 })
  422. expectRepeatedFailure(
  423. meter(),
  424. overlapping,
  425. /arrived before turn 1\/step 1 ended/,
  426. )
  427. const late = new Session(SessionId('late-assistant'))
  428. late.append('step/start', { turn: 1, step: 1 })
  429. appendHeader(late, header('deepseek-v4-flash'))
  430. late.append('step/end', { turn: 1, step: 1 })
  431. late.append('assistant/message', {
  432. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  433. turn: 1,
  434. step: 1,
  435. content: [],
  436. }, { surfaceOp: 'append', sourceEventSeqs: [] })
  437. expectRepeatedFailure(
  438. meter(),
  439. late,
  440. /no matching step\/start/,
  441. )
  442. const mismatchedEnd = new Session(SessionId('mismatched-end'))
  443. mismatchedEnd.append('step/start', { turn: 1, step: 1 })
  444. mismatchedEnd.append('step/end', { turn: 1, step: 2 })
  445. expectRepeatedFailure(
  446. meter(),
  447. mismatchedEnd,
  448. /step\/end .* no matching step\/start/,
  449. )
  450. })
  451. it('rejects invalid assistant provenance', () => {
  452. const cases: Array<{
  453. name: string
  454. appendSource(session: Session): number[]
  455. pattern: RegExp
  456. }> = [
  457. {
  458. name: 'non-chunk',
  459. appendSource(session) {
  460. return [session.append('user/message', {
  461. content: [{ type: 'text', text: 'x' }],
  462. source: { kind: 'user' },
  463. }, { surfaceOp: 'append' }).seq]
  464. },
  465. pattern: /is not assistant\/chunk/,
  466. },
  467. {
  468. name: 'wrong-step',
  469. appendSource(session) {
  470. return [session.append('assistant/chunk', {
  471. turn: 1,
  472. step: 2,
  473. chunk: { type: 'finish', reason: { kind: 'stop' } },
  474. }).seq]
  475. },
  476. pattern: /belongs to another step/,
  477. },
  478. ]
  479. for (const testCase of cases) {
  480. const session = new Session(SessionId(`bad-source-${testCase.name}`))
  481. session.append('step/start', { turn: 1, step: 1 })
  482. appendHeader(session, header('deepseek-v4-flash'))
  483. const sourceEventSeqs = testCase.appendSource(session)
  484. session.append('assistant/message', {
  485. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  486. turn: 1,
  487. step: 1,
  488. content: [{ type: 'text', text: 'bad' }],
  489. usage: { inputTokens: 1, outputTokens: 1 },
  490. }, { surfaceOp: 'append', sourceEventSeqs })
  491. expect(() => meter().measure(session)).toThrow(testCase.pattern)
  492. }
  493. })
  494. it('rejects repeated and non-earlier assistant provenance', () => {
  495. const duplicate = new Session(SessionId('duplicate-source'))
  496. duplicate.append('step/start', { turn: 1, step: 1 })
  497. appendHeader(duplicate, header('deepseek-v4-flash'))
  498. const source = duplicate.append('assistant/chunk', {
  499. turn: 1,
  500. step: 1,
  501. chunk: { type: 'finish', reason: { kind: 'stop' } },
  502. }).seq
  503. appendUnchecked(duplicate, {
  504. type: 'assistant/message',
  505. seq: duplicate.seq,
  506. time: 0,
  507. data: {
  508. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  509. turn: 1,
  510. step: 1,
  511. content: [],
  512. usage: { inputTokens: 1, outputTokens: 0 },
  513. },
  514. surfaceOp: 'append',
  515. sourceEventSeqs: [source, source],
  516. })
  517. expect(() => meter().measure(duplicate)).toThrow(/repeats source seq/)
  518. const future = new Session(SessionId('future-source'))
  519. future.append('step/start', { turn: 1, step: 1 })
  520. appendHeader(future, header('deepseek-v4-flash'))
  521. appendUnchecked(future, {
  522. type: 'assistant/message',
  523. seq: future.seq,
  524. time: 0,
  525. data: {
  526. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  527. turn: 1,
  528. step: 1,
  529. content: [],
  530. usage: { inputTokens: 1, outputTokens: 0 },
  531. },
  532. surfaceOp: 'append',
  533. sourceEventSeqs: [99],
  534. })
  535. expect(() => meter().measure(future)).toThrow(/is not earlier/)
  536. })
  537. it('does not partially apply a malformed assistant replacement', () => {
  538. const session = new Session(SessionId('transactional-replace'))
  539. session.append('user/message', {
  540. content: [{ type: 'text', text: 'head' }],
  541. source: { kind: 'user' },
  542. }, { surfaceOp: 'append' })
  543. appendHeader(session, header('deepseek-v4-flash'))
  544. const head = session.events[0]!.seq
  545. session.append('assistant/message', {
  546. provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
  547. turn: 1,
  548. step: 1,
  549. content: [{ type: 'text', text: 'replacement' }],
  550. }, { surfaceOp: { op: 'replace', start: head, end: head }, sourceEventSeqs: [head] })
  551. expectRepeatedFailure(
  552. meter(),
  553. session,
  554. /no matching step\/start/,
  555. )
  556. })
  557. it('rejects corrupt replacement ranges without advancing the replay cursor', () => {
  558. const session = new Session(SessionId('bad-replace'))
  559. const head = session.append('user/message', {
  560. content: [{ type: 'text', text: 'head' }],
  561. source: { kind: 'user' },
  562. }, { surfaceOp: 'append' }).seq
  563. appendUnchecked(session, {
  564. type: 'user/message',
  565. seq: session.seq,
  566. time: 0,
  567. data: {
  568. content: [{ type: 'text', text: 'bad' }],
  569. source: { kind: 'user' },
  570. },
  571. surfaceOp: { op: 'replace', start: 99, end: 99 },
  572. sourceEventSeqs: [head],
  573. })
  574. expectRepeatedFailure(meter(), session, /invalid current range/)
  575. })
  576. it('handles earlier-reader catch-up, eager observation, and service reload', async () => {
  577. const ctx = new Context()
  578. await ctx.plugin(SessionStore)
  579. let activeMeter: TokenMeterService | undefined
  580. const revisions: number[] = []
  581. ctx.on('session/event', (session) => {
  582. if (activeMeter !== undefined) revisions.push(activeMeter.measure(session).logRevision)
  583. })
  584. const firstFiber = await ctx.plugin(TokenMeterService)
  585. activeMeter = ctx.tokenMeter
  586. const session = ctx.sessions.create(SessionId('listener-order'))
  587. activeMeter.measure(session)
  588. session.append('user/message', {
  589. content: [{ type: 'text', text: 'one' }],
  590. source: { kind: 'user' },
  591. }, { surfaceOp: 'append' })
  592. expect(revisions).toEqual([1])
  593. expect(activeMeter.measure(session).logRevision).toBe(1)
  594. await firstFiber.dispose()
  595. const secondFiber = await ctx.plugin(TokenMeterService)
  596. activeMeter = ctx.tokenMeter
  597. expect(activeMeter.measure(session).logRevision).toBe(1)
  598. await secondFiber.dispose()
  599. })
  600. })