index.ts 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046
  1. /**
  2. * Concrete session-query service with SQLite FTS5 over the live-preferred corpus.
  3. *
  4. * @module @deepseek-ai/dsh-session-query-sqlite
  5. */
  6. import { createHash, randomUUID } from 'node:crypto'
  7. import { DatabaseSync } from 'node:sqlite'
  8. import { Context, Service, type Fiber } from 'cordis'
  9. import z from 'schemastery'
  10. import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
  11. import type SessionPersistence from '@deepseek-ai/dsh-session-persistence'
  12. import type {
  13. SessionPersistenceRevision,
  14. SessionPersistenceSnapshot,
  15. } from '@deepseek-ai/dsh-session-persistence'
  16. import SessionQueryService, {
  17. SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY,
  18. SESSION_QUERY_READ_WINDOW_MAX,
  19. SessionQueryError,
  20. SessionSearchCursor,
  21. assertSessionHeadersCompatible,
  22. buildSessionEventSearchDocuments,
  23. } from '@deepseek-ai/dsh-session-query'
  24. import type {
  25. Config as SessionQueryConfig,
  26. SessionEventSearchDocument,
  27. SessionEventSearchHit,
  28. SessionEventSearchPage,
  29. SessionEventSearchRequest,
  30. SessionSearchExecContext,
  31. SessionSearchHit,
  32. SessionSearchCursor as SessionSearchCursorValue,
  33. SessionSearchPage,
  34. SessionSearchRequest,
  35. } from '@deepseek-ai/dsh-session-query'
  36. import {
  37. type JournalMode,
  38. openSearchDatabase,
  39. } from './schema.ts'
  40. import {
  41. type NormalizedEventRequest,
  42. type NormalizedSessionRequest,
  43. FTS_HIGHLIGHT_END,
  44. FTS_HIGHLIGHT_START,
  45. assertFts5OuterPredicateCount,
  46. assertPortableBindingCount,
  47. buildEventWhere,
  48. buildSessionWhere,
  49. makeSnippet,
  50. normalizeEventRequest,
  51. normalizeSessionRequest,
  52. quoteFtsData,
  53. requestFingerprint,
  54. sanitizeFtsText,
  55. SQLITE_MAX_PAGE_LIMIT,
  56. } from './query.ts'
  57. export {
  58. SESSION_QUERY_SQLITE_APPLICATION_ID,
  59. SESSION_QUERY_SQLITE_SCHEMA_VERSION,
  60. type JournalMode,
  61. } from './schema.ts'
  62. /** Default result page size. */
  63. export const SESSION_QUERY_SQLITE_DEFAULT_LIMIT = 20
  64. /** Maximum accepted result page size. */
  65. export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100
  66. /** Default maximum snippet length in Unicode code points. */
  67. export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240
  68. // One transient source change gets a retry; repeated churn fails rather than monopolizing the queue.
  69. const STABLE_OBSERVATION_ATTEMPTS = 2
  70. /** Combined session-query configuration backed by SQLite full-text search. */
  71. export interface Config extends SessionQueryConfig {
  72. /**
  73. * Dedicated derived-index path; `:memory:` is supported for tests. Missing
  74. * directories and database files are created owner-only on POSIX filesystems;
  75. * existing modes are preserved.
  76. */
  77. path: string
  78. /** SQLite journal mode. Defaults to `wal`. */
  79. journalMode?: JournalMode
  80. /** Page size when a request omits `limit`. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 20. */
  81. defaultLimit?: number
  82. /** Largest accepted page size. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 100. */
  83. maxLimit?: number
  84. /** Maximum snippet length in Unicode code points. Defaults to 240. */
  85. snippetChars?: number
  86. /** Maximum concurrent persisted-log inspections in one inherited batch read. Defaults to 4. */
  87. persistedInspectConcurrency?: number
  88. }
  89. interface ResolvedConfig {
  90. path: string
  91. journalMode: JournalMode
  92. defaultLimit: number
  93. maxLimit: number
  94. snippetChars: number
  95. readWindowMax: number
  96. persistedInspectConcurrency: number
  97. }
  98. interface ObservedSession {
  99. header: SessionHeader
  100. documents: SessionEventSearchDocument[]
  101. fingerprint: string
  102. }
  103. interface ObservedPersistedSession {
  104. header: SessionHeader
  105. revision: SessionPersistenceRevision
  106. loaded?: ObservedSession
  107. }
  108. interface PersistenceBinding {
  109. readonly identity: symbol
  110. readonly service?: SessionPersistence
  111. }
  112. interface Observation {
  113. persistenceBinding: PersistenceBinding
  114. persisted: Map<SessionId, ObservedPersistedSession>
  115. live: Map<SessionId, ObservedSession>
  116. }
  117. interface IndexedPersistedRow {
  118. id: string
  119. revision: string
  120. generation: number
  121. }
  122. interface IndexedLiveRow {
  123. id: string
  124. fingerprint: string
  125. persisted: number
  126. generation: number
  127. }
  128. interface SessionHeaderRow {
  129. session_id: string
  130. version: number
  131. created_at: number
  132. cwd: string | null
  133. parent_session: string | null
  134. seed_length: number | null
  135. delegation_depth: number | null
  136. }
  137. interface SearchRow extends SessionHeaderRow {
  138. live: number
  139. persisted: number
  140. seq: number
  141. type: string
  142. time: number
  143. surface: string
  144. marked_text: string
  145. match_count: number
  146. document_length: number
  147. }
  148. interface CursorPayload {
  149. version: 1
  150. instance: string
  151. scope: 'sessions' | 'events'
  152. fingerprint: string
  153. generation: string
  154. offset: number
  155. }
  156. /** Concrete SQLite owner of the combined `ctx.sessionQuery` service. */
  157. export class SessionQuerySqlite extends SessionQueryService {
  158. static override inject = ['sessions']
  159. static Config: z<Config> = z.object({
  160. path: z.string().required(),
  161. journalMode: z.union(['wal', 'delete', 'truncate', 'persist'] as const).default('wal'),
  162. defaultLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT),
  163. maxLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_MAX_LIMIT),
  164. snippetChars: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_SNIPPET_CHARS),
  165. readWindowMax: z.number().step(1).min(0).default(SESSION_QUERY_READ_WINDOW_MAX),
  166. persistedInspectConcurrency: z.number()
  167. .step(1)
  168. .min(1)
  169. .max(Number.MAX_SAFE_INTEGER)
  170. .default(SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY),
  171. })
  172. /** Validated and defaulted backend configuration. */
  173. readonly config: ResolvedConfig
  174. private readonly _instance = randomUUID()
  175. private readonly _ready: Promise<void>
  176. private _db: DatabaseSync | undefined
  177. private _persistenceBinding: PersistenceBinding = { identity: Symbol() }
  178. private _lastPersistenceIdentity: symbol | undefined
  179. private _persistenceEpoch = 0
  180. private _globalGeneration = 0
  181. private _localGeneration = 0
  182. private _tail: Promise<void> = Promise.resolve()
  183. private _closed = false
  184. private _closePromise: Promise<void> | undefined
  185. private readonly _optionalPersistenceFiber: Fiber
  186. constructor(ctx: Context, config: Config) {
  187. // The assignment expression resolves before the base constructor can
  188. // register `ctx.sessionQuery`; keep that same validated value afterward.
  189. super(ctx, config = resolveConfig(config))
  190. this.config = config as ResolvedConfig
  191. this._ready = this._open()
  192. this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => {
  193. const service = childCtx.sessionPersistence
  194. const binding = { identity: Symbol(), service }
  195. this._persistenceBinding = binding
  196. childCtx.effect(() => () => {
  197. /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */
  198. if (this._persistenceBinding !== binding) return
  199. this._persistenceBinding = { identity: Symbol() }
  200. }, 'sessionQuerySqlite.persistenceBinding')
  201. })
  202. ctx.effect(() => {
  203. return () => this._optionalPersistenceFiber.dispose()
  204. }, 'sessionQuerySqlite.optionalPersistence')
  205. ctx.effect(() => async () => this.close(), 'sessionQuerySqlite.close')
  206. }
  207. /** Open the index before Cordis publishes this combined service as active. */
  208. protected async [Service.init](): Promise<void> {
  209. await this._ensureReady(undefined)
  210. }
  211. override async searchSessions(
  212. request: SessionSearchRequest,
  213. exec?: SessionSearchExecContext,
  214. ): Promise<SessionSearchPage<SessionSearchHit>> {
  215. const normalized = normalizeSessionRequest(request, this.config)
  216. const signal = exec?.signal
  217. return this._serialized(signal, async () => {
  218. await this._ensureReady(signal)
  219. const persistenceBinding = await this._reconcile(signal)
  220. assertNotAborted(signal)
  221. const generation = String(this._globalGeneration)
  222. const fingerprint = requestFingerprint(normalized)
  223. const offset = normalized.cursor === undefined
  224. ? 0
  225. : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation)
  226. const rows = this._querySessions(normalized, offset, persistenceBinding)
  227. return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({
  228. version: 1,
  229. instance: this._instance,
  230. scope: 'sessions',
  231. fingerprint,
  232. generation,
  233. offset: cursorOffset,
  234. }), offset)
  235. })
  236. }
  237. override async searchEvents(
  238. request: SessionEventSearchRequest,
  239. exec?: SessionSearchExecContext,
  240. ): Promise<SessionEventSearchPage> {
  241. const normalized = normalizeEventRequest(request, this.config)
  242. const signal = exec?.signal
  243. return this._serialized(signal, async () => {
  244. await this._ensureReady(signal)
  245. const persistenceBinding = await this._reconcile(signal)
  246. assertNotAborted(signal)
  247. const target = this._targetObservation(normalized.sessionId, persistenceBinding)
  248. const fingerprint = requestFingerprint(normalized)
  249. const offset = normalized.cursor === undefined
  250. ? 0
  251. : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, target.generation)
  252. const rows = this._queryEvents(normalized, offset, persistenceBinding)
  253. return {
  254. session: target.header,
  255. ...page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({
  256. version: 1,
  257. instance: this._instance,
  258. scope: 'events',
  259. fingerprint,
  260. generation: target.generation,
  261. offset: cursorOffset,
  262. }), offset),
  263. }
  264. })
  265. }
  266. /** Close the database after every accepted operation reaches quiescence. */
  267. close(): Promise<void> {
  268. this._closePromise ??= this._close()
  269. return this._closePromise
  270. }
  271. private async _close(): Promise<void> {
  272. this._closed = true
  273. await this._tail
  274. try {
  275. await this._ready
  276. } catch {
  277. // Opening already closed a partially-created handle; disposal only waits.
  278. }
  279. this._db?.close()
  280. this._db = undefined
  281. }
  282. private async _open(): Promise<void> {
  283. this._db = await openSearchDatabase(this.config.path, this.config.journalMode)
  284. const state = this._db.prepare(
  285. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  286. ).get() as { global_generation: number }
  287. this._globalGeneration = state.global_generation
  288. this._localGeneration = state.global_generation
  289. }
  290. private async _ensureReady(signal: AbortSignal | undefined): Promise<void> {
  291. try {
  292. await waitWithAbort(this._ready, signal)
  293. } catch (error: unknown) {
  294. if (isAbort(error)) throw error
  295. throw new SessionQueryError(
  296. `session-search SQLite index failed to open: ${errorMessage(error)}`,
  297. 'SESSION_QUERY_INDEX_FAILED',
  298. { cause: error },
  299. )
  300. }
  301. }
  302. private async _serialized<T>(signal: AbortSignal | undefined, operation: () => Promise<T>): Promise<T> {
  303. if (this._isClosed()) throw indexClosed()
  304. let release!: () => void
  305. const gate = new Promise<void>((resolve) => { release = resolve })
  306. const prior = this._tail
  307. this._tail = prior.then(() => gate)
  308. try {
  309. await waitWithAbort(prior, signal)
  310. } catch (error: unknown) {
  311. release()
  312. throw error
  313. }
  314. if (this._isClosed()) {
  315. release()
  316. throw indexClosed()
  317. }
  318. try {
  319. assertNotAborted(signal)
  320. return await operation()
  321. } finally {
  322. release()
  323. }
  324. }
  325. private async _reconcile(signal: AbortSignal | undefined): Promise<PersistenceBinding> {
  326. assertNotAborted(signal)
  327. const db = this._requireDb()
  328. const persistedRows = db.prepare(
  329. 'SELECT id, revision, generation FROM persisted_sessions',
  330. ).all() as unknown as IndexedPersistedRow[]
  331. const liveRows = db.prepare(
  332. 'SELECT id, fingerprint, persisted, generation FROM temp.live_sessions',
  333. ).all() as unknown as IndexedLiveRow[]
  334. const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row]))
  335. const liveById = new Map(liveRows.map(row => [row.id as SessionId, row]))
  336. const observation = await this._observeStable(persistedById, signal)
  337. assertNotAborted(signal)
  338. const persistentChanges = observation.persistenceBinding.service === undefined
  339. ? []
  340. : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined)
  341. const persistentDeletes = observation.persistenceBinding.service === undefined
  342. ? []
  343. : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId))
  344. const liveChanges = [...observation.live.values()].filter((entry) => {
  345. const indexed = liveById.get(entry.header.id)
  346. const persisted = observation.persisted.has(entry.header.id) ? 1 : 0
  347. return indexed?.fingerprint !== entry.fingerprint || indexed.persisted !== persisted
  348. })
  349. const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId))
  350. const pointerChanged = this._lastPersistenceIdentity !== undefined
  351. && this._lastPersistenceIdentity !== observation.persistenceBinding.identity
  352. const hasWrites = persistentChanges.length > 0
  353. || persistentDeletes.length > 0
  354. || liveChanges.length > 0
  355. || liveDeletes.length > 0
  356. let nextMainGeneration = this._mainGeneration()
  357. let nextLocalGeneration = this._localGeneration
  358. if (persistentChanges.length > 0 || persistentDeletes.length > 0) nextMainGeneration += 1
  359. const liveReplacements = liveChanges.map((entry) => {
  360. nextLocalGeneration = Math.max(nextLocalGeneration, nextMainGeneration) + 1
  361. return {
  362. entry,
  363. generation: nextLocalGeneration,
  364. persisted: observation.persisted.has(entry.header.id),
  365. }
  366. })
  367. if (hasWrites) {
  368. let began = false
  369. try {
  370. db.exec('BEGIN IMMEDIATE')
  371. began = true
  372. for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId)
  373. for (const entry of persistentChanges) {
  374. /* v8 ignore next -- observation loads every entry whose revision differs */
  375. if (entry.loaded === undefined) throw new Error(`missing loaded revision for session "${entry.header.id}"`)
  376. this._replacePersistedSession(entry.loaded, entry.revision, nextMainGeneration)
  377. }
  378. if (persistentChanges.length > 0 || persistentDeletes.length > 0) {
  379. db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration)
  380. }
  381. for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId)
  382. for (const { entry, generation, persisted } of liveReplacements) {
  383. this._replaceLiveSession(entry, generation, persisted)
  384. }
  385. db.exec('COMMIT')
  386. } catch (error: unknown) {
  387. /* v8 ignore next -- a BEGIN failure has no transaction to roll back; the common wrapper still reports it. */
  388. if (began) {
  389. /* v8 ignore next 5 -- ROLLBACK failure requires a SQLite double fault; the original failure remains actionable. */
  390. try {
  391. db.exec('ROLLBACK')
  392. } catch {
  393. // The original SQLite failure remains the actionable cause.
  394. }
  395. }
  396. throw new SessionQueryError(
  397. `session-search reconciliation failed: ${errorMessage(error)}`,
  398. 'SESSION_QUERY_INDEX_FAILED',
  399. { cause: error },
  400. )
  401. }
  402. }
  403. if (hasWrites || pointerChanged) this._globalGeneration += 1
  404. if (pointerChanged) this._persistenceEpoch += 1
  405. this._localGeneration = nextLocalGeneration
  406. this._lastPersistenceIdentity = observation.persistenceBinding.identity
  407. return observation.persistenceBinding
  408. }
  409. private async _observeStable(
  410. indexed: ReadonlyMap<SessionId, IndexedPersistedRow>,
  411. signal: AbortSignal | undefined,
  412. ): Promise<Observation> {
  413. for (let attempt = 0; attempt < STABLE_OBSERVATION_ATTEMPTS; attempt += 1) {
  414. assertNotAborted(signal)
  415. const persistenceBinding = this._persistenceBinding
  416. const persistence = persistenceBinding.service
  417. const initiallyLive = new Set(this.ctx.sessions.list().map(session => session.id))
  418. let persisted = new Map<SessionId, ObservedPersistedSession>()
  419. if (persistence !== undefined) {
  420. try {
  421. const canReuseIndexed = this._lastPersistenceIdentity === undefined
  422. || this._lastPersistenceIdentity === persistenceBinding.identity
  423. const before = await persistence.listSnapshots(signal)
  424. assertNotAborted(signal)
  425. persisted = materializePersistenceSnapshots(before)
  426. for (const entry of persisted.values()) {
  427. if (canReuseIndexed && indexed.get(entry.header.id)?.revision === entry.revision) continue
  428. // Skip work already shadowed by a live owner. `inspect()` is
  429. // non-mutating, so an owner attaching after this check cannot cause
  430. // crash-repair side effects; the live-membership retry below makes
  431. // the returned observation live-preferred.
  432. if (initiallyLive.has(entry.header.id) || this.ctx.sessions.get(entry.header.id) !== undefined) continue
  433. assertNotAborted(signal)
  434. const loaded = await persistence.inspect(entry.header.id, signal)
  435. assertNotAborted(signal)
  436. assertSessionHeadersCompatible(entry.header, loaded.meta)
  437. entry.loaded = observeSession(loaded.meta, loaded.events)
  438. }
  439. assertNotAborted(signal)
  440. const afterSnapshots = await persistence.listSnapshots(signal)
  441. assertNotAborted(signal)
  442. const after = materializePersistenceSnapshots(afterSnapshots)
  443. if (!samePersistenceSnapshots(persisted, after)) continue
  444. if (this._persistenceBinding !== persistenceBinding) continue
  445. } catch (error: unknown) {
  446. if (isAbort(error) || signal?.aborted) {
  447. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', {
  448. cause: error,
  449. })
  450. }
  451. if (this._persistenceBinding !== persistenceBinding) continue
  452. if (error instanceof SessionQueryError) throw error
  453. throw new SessionQueryError(
  454. `session-search persistence observation failed: ${errorMessage(error)}`,
  455. 'SESSION_QUERY_PERSISTENCE_FAILED',
  456. { cause: error },
  457. )
  458. }
  459. }
  460. const live = new Map<SessionId, ObservedSession>()
  461. for (const session of this.ctx.sessions.list()) {
  462. const observed = observeLive(session)
  463. const durable = persisted.get(session.id)
  464. if (durable !== undefined) assertSessionHeadersCompatible(observed.header, durable.header)
  465. live.set(session.id, observed)
  466. }
  467. if (!sameSessionIds(initiallyLive, live)) continue
  468. return { persistenceBinding, persisted, live }
  469. }
  470. throw new SessionQueryError(
  471. 'session-search persistence observation did not stabilize after one retry',
  472. 'SESSION_QUERY_PERSISTENCE_FAILED',
  473. )
  474. }
  475. private _mainGeneration(): number {
  476. const row = this._requireDb().prepare(
  477. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  478. ).get() as { global_generation: number }
  479. return row.global_generation
  480. }
  481. private _deleteSession(source: 'persisted' | 'live', id: SessionId): void {
  482. const db = this._requireDb()
  483. if (source === 'persisted') {
  484. db.prepare('DELETE FROM persisted_docs WHERE session_id = ?').run(id)
  485. db.prepare('DELETE FROM persisted_sessions WHERE id = ?').run(id)
  486. } else {
  487. db.prepare('DELETE FROM temp.live_docs WHERE session_id = ?').run(id)
  488. db.prepare('DELETE FROM temp.live_sessions WHERE id = ?').run(id)
  489. }
  490. }
  491. private _replacePersistedSession(
  492. entry: ObservedSession,
  493. revision: SessionPersistenceRevision,
  494. generation: number,
  495. ): void {
  496. this._deleteSession('persisted', entry.header.id)
  497. const db = this._requireDb()
  498. db.prepare(`
  499. INSERT INTO persisted_sessions
  500. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, revision, generation)
  501. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
  502. `).run(
  503. entry.header.id,
  504. entry.header.version,
  505. entry.header.createdAt,
  506. entry.header.cwd ?? null,
  507. entry.header.parentSession ?? null,
  508. entry.header.seedLength ?? null,
  509. entry.header.delegationDepth ?? null,
  510. revision,
  511. generation,
  512. )
  513. const insert = db.prepare(`
  514. INSERT INTO persisted_docs (text, session_id, seq, type, time, surface, codepoint_length)
  515. VALUES (?, ?, ?, ?, ?, ?, ?)
  516. `)
  517. for (const document of entry.documents) {
  518. const text = sanitizeFtsText(document.text)
  519. insert.run(
  520. text,
  521. document.sessionId,
  522. document.seq,
  523. document.type,
  524. document.time,
  525. document.surface,
  526. Array.from(text).length,
  527. )
  528. }
  529. }
  530. private _replaceLiveSession(entry: ObservedSession, generation: number, persisted: boolean): void {
  531. this._deleteSession('live', entry.header.id)
  532. const db = this._requireDb()
  533. db.prepare(`
  534. INSERT INTO temp.live_sessions
  535. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, fingerprint, persisted, generation)
  536. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  537. `).run(
  538. entry.header.id,
  539. entry.header.version,
  540. entry.header.createdAt,
  541. entry.header.cwd ?? null,
  542. entry.header.parentSession ?? null,
  543. entry.header.seedLength ?? null,
  544. entry.header.delegationDepth ?? null,
  545. entry.fingerprint,
  546. persisted ? 1 : 0,
  547. generation,
  548. )
  549. const insert = db.prepare(`
  550. INSERT INTO temp.live_docs (text, session_id, seq, type, time, surface, codepoint_length)
  551. VALUES (?, ?, ?, ?, ?, ?, ?)
  552. `)
  553. for (const document of entry.documents) {
  554. const text = sanitizeFtsText(document.text)
  555. insert.run(
  556. text,
  557. document.sessionId,
  558. document.seq,
  559. document.type,
  560. document.time,
  561. document.surface,
  562. Array.from(text).length,
  563. )
  564. }
  565. }
  566. private _querySessions(
  567. request: NormalizedSessionRequest,
  568. offset: number,
  569. persistenceBinding: PersistenceBinding,
  570. ): SearchRow[] {
  571. const selected = selectedDocumentsSql()
  572. const sessionWhere = buildSessionWhere(request.sessionFilters)
  573. const eventWhere = buildEventWhere(request.eventFilters)
  574. assertFts5OuterPredicateCount(sessionWhere.predicateCount + eventWhere.predicateCount)
  575. const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ')
  576. const bindings = [
  577. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  578. ...sessionWhere.params,
  579. ...eventWhere.params,
  580. request.limit + 1,
  581. offset,
  582. ]
  583. assertPortableBindingCount(bindings.length)
  584. return this._requireDb().prepare(`
  585. ${selected.sql},
  586. filtered AS (
  587. SELECT * FROM matched ${where.length === 0 ? '' : `WHERE ${where}`}
  588. ),
  589. ranked AS (
  590. SELECT *, ROW_NUMBER() OVER (
  591. PARTITION BY session_id
  592. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  593. ) AS event_rank
  594. FROM filtered
  595. )
  596. SELECT * FROM ranked
  597. WHERE event_rank = 1
  598. ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC
  599. LIMIT ? OFFSET ?
  600. `).all(...bindings) as unknown as SearchRow[]
  601. }
  602. private _queryEvents(
  603. request: NormalizedEventRequest,
  604. offset: number,
  605. persistenceBinding: PersistenceBinding,
  606. ): SearchRow[] {
  607. const selected = selectedDocumentsSql()
  608. const eventWhere = buildEventWhere(request.filters)
  609. assertFts5OuterPredicateCount(1 + eventWhere.predicateCount)
  610. const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ')
  611. const bindings = [
  612. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  613. request.sessionId,
  614. ...eventWhere.params,
  615. request.limit + 1,
  616. offset,
  617. ]
  618. assertPortableBindingCount(bindings.length)
  619. return this._requireDb().prepare(`
  620. ${selected.sql}
  621. SELECT * FROM matched
  622. WHERE ${where}
  623. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  624. LIMIT ? OFFSET ?
  625. `).all(...bindings) as unknown as SearchRow[]
  626. }
  627. private _targetObservation(
  628. sessionId: SessionId,
  629. persistenceBinding: PersistenceBinding,
  630. ): { header: SessionHeader; generation: string } {
  631. const db = this._requireDb()
  632. const live = db.prepare(
  633. `SELECT
  634. id AS session_id, version, created_at, cwd, parent_session, seed_length, delegation_depth, generation
  635. FROM temp.live_sessions
  636. WHERE id = ?`,
  637. ).get(sessionId) as (SessionHeaderRow & { generation: number }) | undefined
  638. if (live !== undefined) {
  639. return { header: rowHeader(live), generation: `live:${live.generation}` }
  640. }
  641. if (persistenceBinding.service !== undefined) {
  642. const persisted = db.prepare(
  643. `SELECT
  644. id AS session_id, version, created_at, cwd, parent_session, seed_length, delegation_depth, generation
  645. FROM persisted_sessions
  646. WHERE id = ?`,
  647. ).get(sessionId) as (SessionHeaderRow & { generation: number }) | undefined
  648. if (persisted !== undefined) {
  649. return {
  650. header: rowHeader(persisted),
  651. generation: `persisted:${this._persistenceEpoch}:${persisted.generation}`,
  652. }
  653. }
  654. }
  655. throw new SessionQueryError(
  656. `session "${sessionId}" not found`,
  657. 'SESSION_QUERY_SESSION_NOT_FOUND',
  658. )
  659. }
  660. private _sessionHit(row: SearchRow): SessionSearchHit {
  661. return {
  662. header: rowHeader(row),
  663. live: row.live === 1,
  664. persisted: row.persisted === 1,
  665. bestMatch: this._eventHit(row),
  666. }
  667. }
  668. private _eventHit(row: SearchRow): SessionEventSearchHit {
  669. return {
  670. sessionId: row.session_id as SessionId,
  671. seq: row.seq,
  672. type: row.type as SessionEventSearchHit['type'],
  673. time: row.time,
  674. surface: row.surface as SessionEventSearchHit['surface'],
  675. snippet: makeSnippet(row.marked_text, this.config.snippetChars),
  676. }
  677. }
  678. private _requireDb(): DatabaseSync {
  679. /* v8 ignore next -- callers await `_ready`; this guards lifecycle misuse */
  680. if (this._db === undefined) throw indexClosed()
  681. return this._db
  682. }
  683. private _isClosed(): boolean {
  684. return this._closed
  685. }
  686. }
  687. function selectedDocumentsSql(): { sql: string } {
  688. return {
  689. sql: `WITH candidates AS (
  690. SELECT
  691. pd.session_id AS session_id,
  692. ps.version AS version,
  693. ps.created_at AS created_at,
  694. ps.cwd AS cwd,
  695. ps.parent_session AS parent_session,
  696. ps.seed_length AS seed_length,
  697. ps.delegation_depth AS delegation_depth,
  698. 0 AS live,
  699. 1 AS persisted,
  700. CAST(pd.seq AS INTEGER) AS seq,
  701. pd.type AS type,
  702. CAST(pd.time AS INTEGER) AS time,
  703. pd.surface AS surface,
  704. highlight(persisted_docs, 0, ?, ?) AS marked_text,
  705. CAST(pd.codepoint_length AS INTEGER) AS document_length
  706. FROM persisted_docs AS pd
  707. JOIN persisted_sessions AS ps ON ps.id = pd.session_id
  708. WHERE persisted_docs MATCH ?
  709. AND ? = 1
  710. AND NOT EXISTS (SELECT 1 FROM temp.live_sessions AS ls WHERE ls.id = pd.session_id)
  711. UNION ALL
  712. SELECT
  713. ld.session_id AS session_id,
  714. ls.version AS version,
  715. ls.created_at AS created_at,
  716. ls.cwd AS cwd,
  717. ls.parent_session AS parent_session,
  718. ls.seed_length AS seed_length,
  719. ls.delegation_depth AS delegation_depth,
  720. 1 AS live,
  721. CASE WHEN ? = 1 THEN ls.persisted ELSE 0 END AS persisted,
  722. CAST(ld.seq AS INTEGER) AS seq,
  723. ld.type AS type,
  724. CAST(ld.time AS INTEGER) AS time,
  725. ld.surface AS surface,
  726. highlight(live_docs, 0, ?, ?) AS marked_text,
  727. CAST(ld.codepoint_length AS INTEGER) AS document_length
  728. FROM temp.live_docs AS ld
  729. JOIN temp.live_sessions AS ls ON ls.id = ld.session_id
  730. WHERE live_docs MATCH ?
  731. ), matched AS (
  732. SELECT *,
  733. (
  734. length(CAST(marked_text AS BLOB))
  735. - length(CAST(replace(marked_text, ?, '') AS BLOB))
  736. ) / ? AS match_count
  737. FROM candidates
  738. )`,
  739. }
  740. }
  741. function selectedDocumentsParams(query: string, persistenceVisible: boolean): Array<string | number> {
  742. const expression = quoteFtsData(query)
  743. const visible = persistenceVisible ? 1 : 0
  744. return [
  745. FTS_HIGHLIGHT_START,
  746. FTS_HIGHLIGHT_END,
  747. expression,
  748. visible,
  749. visible,
  750. FTS_HIGHLIGHT_START,
  751. FTS_HIGHLIGHT_END,
  752. expression,
  753. FTS_HIGHLIGHT_START,
  754. Buffer.byteLength(FTS_HIGHLIGHT_START, 'utf8'),
  755. ]
  756. }
  757. function observeLive(session: Session): ObservedSession {
  758. return observeSession(session.header, session.events)
  759. }
  760. function observeSession(header: SessionHeader, events: readonly SessionEvent[]): ObservedSession {
  761. const detachedHeader = structuredClone(header)
  762. const detachedEvents = events.map(event => structuredClone(event))
  763. return {
  764. header: detachedHeader,
  765. documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents),
  766. fingerprint: createHash('sha256')
  767. .update(JSON.stringify({ header: detachedHeader, events: detachedEvents }))
  768. .digest('base64url'),
  769. }
  770. }
  771. function materializePersistenceSnapshots(
  772. snapshots: readonly SessionPersistenceSnapshot[],
  773. ): Map<SessionId, ObservedPersistedSession> {
  774. if (!isRuntimeArray(snapshots)) throw new Error('persistence snapshots must be an array')
  775. const result = new Map<SessionId, ObservedPersistedSession>()
  776. for (const snapshot of snapshots) {
  777. if (typeof snapshot.revision !== 'string') {
  778. throw new Error('persistence snapshot revision must be a string')
  779. }
  780. const header = structuredClone(snapshot.header)
  781. if (result.has(header.id)) {
  782. throw new Error(`persistence listed duplicate session "${header.id}"`)
  783. }
  784. result.set(header.id, { header, revision: snapshot.revision })
  785. }
  786. return result
  787. }
  788. function samePersistenceSnapshots(
  789. before: ReadonlyMap<SessionId, ObservedPersistedSession>,
  790. after: ReadonlyMap<SessionId, ObservedPersistedSession>,
  791. ): boolean {
  792. if (before.size !== after.size) return false
  793. for (const [id, first] of before) {
  794. const second = after.get(id)
  795. if (
  796. second === undefined
  797. || first.revision !== second.revision
  798. || !sameHeader(first.header, second.header)
  799. ) return false
  800. }
  801. return true
  802. }
  803. function sameSessionIds(
  804. before: ReadonlySet<SessionId>,
  805. after: ReadonlyMap<SessionId, ObservedSession>,
  806. ): boolean {
  807. if (before.size !== after.size) return false
  808. for (const id of before) {
  809. if (!after.has(id)) return false
  810. }
  811. return true
  812. }
  813. function sameHeader(a: SessionHeader, b: SessionHeader): boolean {
  814. return a.version === b.version
  815. && a.id === b.id
  816. && a.createdAt === b.createdAt
  817. && a.cwd === b.cwd
  818. && a.parentSession === b.parentSession
  819. && a.seedLength === b.seedLength
  820. && (a.delegationDepth ?? 0) === (b.delegationDepth ?? 0)
  821. }
  822. function rowHeader(row: SessionHeaderRow): SessionHeader {
  823. return {
  824. version: row.version,
  825. id: row.session_id as SessionId,
  826. createdAt: row.created_at,
  827. ...row.cwd === null ? {} : { cwd: row.cwd },
  828. ...row.parent_session === null ? {} : { parentSession: row.parent_session as SessionId },
  829. ...row.seed_length === null ? {} : { seedLength: row.seed_length },
  830. ...row.delegation_depth === null ? {} : { delegationDepth: row.delegation_depth },
  831. }
  832. }
  833. function page<Row, Item>(
  834. rows: readonly Row[],
  835. limit: number,
  836. convert: (row: Row) => Item,
  837. nextCursor: (offset: number) => SessionSearchCursorValue,
  838. offset: number,
  839. ): SessionSearchPage<Item> {
  840. const hasMore = rows.length > limit
  841. return {
  842. items: rows.slice(0, limit).map(convert),
  843. ...hasMore ? { nextCursor: nextCursor(offset + limit) } : {},
  844. }
  845. }
  846. function encodeCursor(payload: CursorPayload): SessionSearchCursorValue {
  847. return SessionSearchCursor(Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'))
  848. }
  849. function decodeCursor(
  850. cursor: SessionSearchCursorValue,
  851. instance: string,
  852. scope: CursorPayload['scope'],
  853. fingerprint: string,
  854. generation: string,
  855. ): number {
  856. let decoded: Partial<CursorPayload>
  857. try {
  858. decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<CursorPayload>
  859. } catch (error: unknown) {
  860. throw invalidCursor(error)
  861. }
  862. if (
  863. decoded.version !== 1
  864. || decoded.instance !== instance
  865. || decoded.scope !== scope
  866. || decoded.fingerprint !== fingerprint
  867. || !Number.isSafeInteger(decoded.offset)
  868. || decoded.offset === undefined
  869. || decoded.offset < 0
  870. ) {
  871. throw invalidCursor(new Error('cursor does not belong to this normalized request'))
  872. }
  873. if (decoded.generation !== generation) {
  874. throw new SessionQueryError(
  875. 'session-search cursor is stale because its relevant corpus changed',
  876. 'SESSION_QUERY_STALE_CURSOR',
  877. )
  878. }
  879. return decoded.offset
  880. }
  881. function invalidCursor(cause: unknown): SessionQueryError {
  882. return new SessionQueryError(
  883. 'session-search cursor is invalid',
  884. 'SESSION_QUERY_INVALID_CURSOR',
  885. { cause },
  886. )
  887. }
  888. function resolveConfig(config: Config): ResolvedConfig {
  889. const resolved: ResolvedConfig = {
  890. path: config.path,
  891. journalMode: config.journalMode ?? 'wal',
  892. defaultLimit: config.defaultLimit ?? SESSION_QUERY_SQLITE_DEFAULT_LIMIT,
  893. maxLimit: config.maxLimit ?? SESSION_QUERY_SQLITE_MAX_LIMIT,
  894. snippetChars: config.snippetChars ?? SESSION_QUERY_SQLITE_SNIPPET_CHARS,
  895. readWindowMax: config.readWindowMax ?? SESSION_QUERY_READ_WINDOW_MAX,
  896. persistedInspectConcurrency: config.persistedInspectConcurrency
  897. ?? SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY,
  898. }
  899. if (typeof resolved.path !== 'string' || resolved.path.trim().length === 0) {
  900. throw invalidConfig('path must not be blank')
  901. }
  902. assertPageLimit('defaultLimit', resolved.defaultLimit)
  903. assertPageLimit('maxLimit', resolved.maxLimit)
  904. assertPositiveInteger('snippetChars', resolved.snippetChars)
  905. if (!Number.isInteger(resolved.readWindowMax) || resolved.readWindowMax < 0) {
  906. throw invalidConfig('readWindowMax must be a non-negative integer')
  907. }
  908. if (
  909. !Number.isSafeInteger(resolved.persistedInspectConcurrency)
  910. || resolved.persistedInspectConcurrency < 1
  911. ) {
  912. throw invalidConfig('persistedInspectConcurrency must be a positive safe integer')
  913. }
  914. if (resolved.defaultLimit > resolved.maxLimit) {
  915. throw invalidConfig('defaultLimit must be less than or equal to maxLimit')
  916. }
  917. const journalModes: readonly string[] = ['wal', 'delete', 'truncate', 'persist']
  918. if (!journalModes.includes(resolved.journalMode)) throw invalidConfig('journalMode is not supported')
  919. return resolved
  920. }
  921. function assertPositiveInteger(name: string, value: number): void {
  922. if (!Number.isInteger(value) || value < 1) throw invalidConfig(`${name} must be a positive integer`)
  923. }
  924. function assertPageLimit(name: string, value: number): void {
  925. if (!Number.isSafeInteger(value) || value < 1 || value > SQLITE_MAX_PAGE_LIMIT) {
  926. throw invalidConfig(`${name} must be an integer between 1 and ${SQLITE_MAX_PAGE_LIMIT}`)
  927. }
  928. }
  929. function invalidConfig(detail: string): SessionQueryError {
  930. return new SessionQueryError(
  931. `session-search SQLite config: ${detail}`,
  932. 'SESSION_QUERY_INVALID_CONFIG',
  933. )
  934. }
  935. function indexClosed(): SessionQueryError {
  936. return new SessionQueryError('session-search SQLite index is closed', 'SESSION_QUERY_INDEX_FAILED')
  937. }
  938. function assertNotAborted(signal: AbortSignal | undefined): void {
  939. if (signal?.aborted) {
  940. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED')
  941. }
  942. }
  943. function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> {
  944. if (signal === undefined) return promise
  945. if (signal.aborted) return Promise.reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  946. return new Promise<T>((resolve, reject) => {
  947. const onAbort = () => {
  948. reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  949. }
  950. signal.addEventListener('abort', onAbort, { once: true })
  951. promise.then(
  952. (value) => {
  953. signal.removeEventListener('abort', onAbort)
  954. resolve(value)
  955. },
  956. (error: unknown) => {
  957. signal.removeEventListener('abort', onAbort)
  958. reject(asError(error))
  959. },
  960. )
  961. })
  962. }
  963. function isAbort(error: unknown): boolean {
  964. return error instanceof SessionQueryError && error.code === 'SESSION_QUERY_ABORTED'
  965. }
  966. function asError(error: unknown): Error {
  967. return error instanceof Error
  968. ? error
  969. : new Error('session-search dependency rejected with a non-Error value', { cause: error })
  970. }
  971. function errorMessage(error: unknown): string {
  972. return error instanceof Error ? error.message : 'unknown error'
  973. }
  974. function isRuntimeArray(value: unknown): boolean {
  975. return Array.isArray(value)
  976. }
  977. export default SessionQuerySqlite