index.ts 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001
  1. /**
  2. * Concrete session-query service with SQLite FTS5 over the live-preferred corpus.
  3. *
  4. * @module @deepseek-ai/dsh-session-query-sqlite
  5. */
  6. import { createHash, randomUUID } from 'node:crypto'
  7. import { DatabaseSync } from 'node:sqlite'
  8. import { Context, Service, type Fiber } from 'cordis'
  9. import z from 'schemastery'
  10. import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
  11. import type SessionPersistence from '@deepseek-ai/dsh-session-persistence'
  12. import type {
  13. SessionPersistenceRevision,
  14. SessionPersistenceSnapshot,
  15. } from '@deepseek-ai/dsh-session-persistence'
  16. import SessionQueryService, {
  17. SESSION_QUERY_READ_WINDOW_MAX,
  18. SessionQueryError,
  19. SessionSearchCursor,
  20. assertSessionHeadersCompatible,
  21. buildSessionEventSearchDocuments,
  22. } from '@deepseek-ai/dsh-session-query'
  23. import type {
  24. Config as SessionQueryConfig,
  25. SessionEventSearchDocument,
  26. SessionEventSearchHit,
  27. SessionEventSearchRequest,
  28. SessionSearchExecContext,
  29. SessionSearchHit,
  30. SessionSearchCursor as SessionSearchCursorValue,
  31. SessionSearchPage,
  32. SessionSearchRequest,
  33. } from '@deepseek-ai/dsh-session-query'
  34. import {
  35. type JournalMode,
  36. openSearchDatabase,
  37. } from './schema.ts'
  38. import {
  39. type NormalizedEventRequest,
  40. type NormalizedSessionRequest,
  41. FTS_HIGHLIGHT_END,
  42. FTS_HIGHLIGHT_START,
  43. assertFts5OuterPredicateCount,
  44. assertPortableBindingCount,
  45. buildEventWhere,
  46. buildSessionWhere,
  47. makeSnippet,
  48. normalizeEventRequest,
  49. normalizeSessionRequest,
  50. quoteFtsData,
  51. requestFingerprint,
  52. sanitizeFtsText,
  53. SQLITE_MAX_PAGE_LIMIT,
  54. } from './query.ts'
  55. export {
  56. SESSION_QUERY_SQLITE_APPLICATION_ID,
  57. SESSION_QUERY_SQLITE_SCHEMA_VERSION,
  58. type JournalMode,
  59. } from './schema.ts'
  60. /** Default result page size. */
  61. export const SESSION_QUERY_SQLITE_DEFAULT_LIMIT = 20
  62. /** Maximum accepted result page size. */
  63. export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100
  64. /** Default maximum snippet length in Unicode code points. */
  65. export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240
  66. // One transient source change gets a retry; repeated churn fails rather than monopolizing the queue.
  67. const STABLE_OBSERVATION_ATTEMPTS = 2
  68. /** Combined session-query configuration backed by SQLite full-text search. */
  69. export interface Config extends SessionQueryConfig {
  70. /**
  71. * Dedicated derived-index path; `:memory:` is supported for tests. Missing
  72. * directories and database files are created owner-only on POSIX filesystems;
  73. * existing modes are preserved.
  74. */
  75. path: string
  76. /** SQLite journal mode. Defaults to `wal`. */
  77. journalMode?: JournalMode
  78. /** Page size when a request omits `limit`. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 20. */
  79. defaultLimit?: number
  80. /** Largest accepted page size. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 100. */
  81. maxLimit?: number
  82. /** Maximum snippet length in Unicode code points. Defaults to 240. */
  83. snippetChars?: number
  84. }
  85. interface ResolvedConfig {
  86. path: string
  87. journalMode: JournalMode
  88. defaultLimit: number
  89. maxLimit: number
  90. snippetChars: number
  91. readWindowMax: number
  92. }
  93. interface ObservedSession {
  94. header: SessionHeader
  95. documents: SessionEventSearchDocument[]
  96. fingerprint: string
  97. }
  98. interface ObservedPersistedSession {
  99. header: SessionHeader
  100. revision: SessionPersistenceRevision
  101. loaded?: ObservedSession
  102. }
  103. interface PersistenceBinding {
  104. readonly identity: symbol
  105. readonly service?: SessionPersistence
  106. }
  107. interface Observation {
  108. persistenceBinding: PersistenceBinding
  109. persisted: Map<SessionId, ObservedPersistedSession>
  110. live: Map<SessionId, ObservedSession>
  111. }
  112. interface IndexedPersistedRow {
  113. id: string
  114. revision: string
  115. generation: number
  116. }
  117. interface IndexedLiveRow {
  118. id: string
  119. fingerprint: string
  120. persisted: number
  121. generation: number
  122. }
  123. interface SearchRow {
  124. session_id: string
  125. version: number
  126. created_at: number
  127. cwd: string | null
  128. parent_session: string | null
  129. seed_length: number | null
  130. delegation_depth: number | null
  131. live: number
  132. persisted: number
  133. seq: number
  134. type: string
  135. time: number
  136. surface: string
  137. marked_text: string
  138. match_count: number
  139. document_length: number
  140. }
  141. interface CursorPayload {
  142. version: 1
  143. instance: string
  144. scope: 'sessions' | 'events'
  145. fingerprint: string
  146. generation: string
  147. offset: number
  148. }
  149. /** Concrete SQLite owner of the combined `ctx.sessionQuery` service. */
  150. export class SessionQuerySqlite extends SessionQueryService {
  151. static override inject = ['sessions']
  152. static Config: z<Config> = z.object({
  153. path: z.string().required(),
  154. journalMode: z.union(['wal', 'delete', 'truncate', 'persist'] as const).default('wal'),
  155. defaultLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT),
  156. maxLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_MAX_LIMIT),
  157. snippetChars: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_SNIPPET_CHARS),
  158. readWindowMax: z.number().step(1).min(0).default(SESSION_QUERY_READ_WINDOW_MAX),
  159. })
  160. /** Validated and defaulted backend configuration. */
  161. readonly config: ResolvedConfig
  162. private readonly _instance = randomUUID()
  163. private readonly _ready: Promise<void>
  164. private _db: DatabaseSync | undefined
  165. private _persistenceBinding: PersistenceBinding = { identity: Symbol() }
  166. private _lastPersistenceIdentity: symbol | undefined
  167. private _persistenceEpoch = 0
  168. private _globalGeneration = 0
  169. private _localGeneration = 0
  170. private _tail: Promise<void> = Promise.resolve()
  171. private _closed = false
  172. private _closePromise: Promise<void> | undefined
  173. private readonly _optionalPersistenceFiber: Fiber
  174. constructor(ctx: Context, config: Config) {
  175. // The assignment expression resolves before the base constructor can
  176. // register `ctx.sessionQuery`; keep that same validated value afterward.
  177. super(ctx, config = resolveConfig(config))
  178. this.config = config as ResolvedConfig
  179. this._ready = this._open()
  180. this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => {
  181. const service = childCtx.sessionPersistence
  182. const binding = { identity: Symbol(), service }
  183. this._persistenceBinding = binding
  184. childCtx.effect(() => () => {
  185. /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */
  186. if (this._persistenceBinding !== binding) return
  187. this._persistenceBinding = { identity: Symbol() }
  188. }, 'sessionQuerySqlite.persistenceBinding')
  189. })
  190. ctx.effect(() => {
  191. return () => this._optionalPersistenceFiber.dispose()
  192. }, 'sessionQuerySqlite.optionalPersistence')
  193. ctx.effect(() => async () => this.close(), 'sessionQuerySqlite.close')
  194. }
  195. /** Open the index before Cordis publishes this combined service as active. */
  196. protected async [Service.init](): Promise<void> {
  197. await this._ensureReady(undefined)
  198. }
  199. override async searchSessions(
  200. request: SessionSearchRequest,
  201. exec?: SessionSearchExecContext,
  202. ): Promise<SessionSearchPage<SessionSearchHit>> {
  203. const normalized = normalizeSessionRequest(request, this.config)
  204. const signal = exec?.signal
  205. return this._serialized(signal, async () => {
  206. await this._ensureReady(signal)
  207. const persistenceBinding = await this._reconcile(signal)
  208. assertNotAborted(signal)
  209. const generation = String(this._globalGeneration)
  210. const fingerprint = requestFingerprint(normalized)
  211. const offset = normalized.cursor === undefined
  212. ? 0
  213. : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation)
  214. const rows = this._querySessions(normalized, offset, persistenceBinding)
  215. return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({
  216. version: 1,
  217. instance: this._instance,
  218. scope: 'sessions',
  219. fingerprint,
  220. generation,
  221. offset: cursorOffset,
  222. }), offset)
  223. })
  224. }
  225. override async searchEvents(
  226. request: SessionEventSearchRequest,
  227. exec?: SessionSearchExecContext,
  228. ): Promise<SessionSearchPage<SessionEventSearchHit>> {
  229. const normalized = normalizeEventRequest(request, this.config)
  230. const signal = exec?.signal
  231. return this._serialized(signal, async () => {
  232. await this._ensureReady(signal)
  233. const persistenceBinding = await this._reconcile(signal)
  234. assertNotAborted(signal)
  235. const generation = this._targetGeneration(normalized.sessionId, persistenceBinding)
  236. const fingerprint = requestFingerprint(normalized)
  237. const offset = normalized.cursor === undefined
  238. ? 0
  239. : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, generation)
  240. const rows = this._queryEvents(normalized, offset, persistenceBinding)
  241. return page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({
  242. version: 1,
  243. instance: this._instance,
  244. scope: 'events',
  245. fingerprint,
  246. generation,
  247. offset: cursorOffset,
  248. }), offset)
  249. })
  250. }
  251. /** Close the database after every accepted operation reaches quiescence. */
  252. close(): Promise<void> {
  253. this._closePromise ??= this._close()
  254. return this._closePromise
  255. }
  256. private async _close(): Promise<void> {
  257. this._closed = true
  258. await this._tail
  259. try {
  260. await this._ready
  261. } catch {
  262. // Opening already closed a partially-created handle; disposal only waits.
  263. }
  264. this._db?.close()
  265. this._db = undefined
  266. }
  267. private async _open(): Promise<void> {
  268. this._db = await openSearchDatabase(this.config.path, this.config.journalMode)
  269. const state = this._db.prepare(
  270. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  271. ).get() as { global_generation: number }
  272. this._globalGeneration = state.global_generation
  273. this._localGeneration = state.global_generation
  274. }
  275. private async _ensureReady(signal: AbortSignal | undefined): Promise<void> {
  276. try {
  277. await waitWithAbort(this._ready, signal)
  278. } catch (error: unknown) {
  279. if (isAbort(error)) throw error
  280. throw new SessionQueryError(
  281. `session-search SQLite index failed to open: ${errorMessage(error)}`,
  282. 'SESSION_QUERY_INDEX_FAILED',
  283. { cause: error },
  284. )
  285. }
  286. }
  287. private async _serialized<T>(signal: AbortSignal | undefined, operation: () => Promise<T>): Promise<T> {
  288. if (this._isClosed()) throw indexClosed()
  289. let release!: () => void
  290. const gate = new Promise<void>((resolve) => { release = resolve })
  291. const prior = this._tail
  292. this._tail = prior.then(() => gate)
  293. try {
  294. await waitWithAbort(prior, signal)
  295. } catch (error: unknown) {
  296. release()
  297. throw error
  298. }
  299. if (this._isClosed()) {
  300. release()
  301. throw indexClosed()
  302. }
  303. try {
  304. assertNotAborted(signal)
  305. return await operation()
  306. } finally {
  307. release()
  308. }
  309. }
  310. private async _reconcile(signal: AbortSignal | undefined): Promise<PersistenceBinding> {
  311. const db = this._requireDb()
  312. const persistedRows = db.prepare(
  313. 'SELECT id, revision, generation FROM persisted_sessions',
  314. ).all() as unknown as IndexedPersistedRow[]
  315. const liveRows = db.prepare(
  316. 'SELECT id, fingerprint, persisted, generation FROM temp.live_sessions',
  317. ).all() as unknown as IndexedLiveRow[]
  318. const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row]))
  319. const liveById = new Map(liveRows.map(row => [row.id as SessionId, row]))
  320. const observation = await this._observeStable(persistedById, signal)
  321. assertNotAborted(signal)
  322. const persistentChanges = observation.persistenceBinding.service === undefined
  323. ? []
  324. : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined)
  325. const persistentDeletes = observation.persistenceBinding.service === undefined
  326. ? []
  327. : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId))
  328. const liveChanges = [...observation.live.values()].filter((entry) => {
  329. const indexed = liveById.get(entry.header.id)
  330. const persisted = observation.persisted.has(entry.header.id) ? 1 : 0
  331. return indexed?.fingerprint !== entry.fingerprint || indexed.persisted !== persisted
  332. })
  333. const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId))
  334. const pointerChanged = this._lastPersistenceIdentity !== undefined
  335. && this._lastPersistenceIdentity !== observation.persistenceBinding.identity
  336. const hasWrites = persistentChanges.length > 0
  337. || persistentDeletes.length > 0
  338. || liveChanges.length > 0
  339. || liveDeletes.length > 0
  340. let nextMainGeneration = this._mainGeneration()
  341. let nextLocalGeneration = this._localGeneration
  342. if (persistentChanges.length > 0 || persistentDeletes.length > 0) nextMainGeneration += 1
  343. const liveReplacements = liveChanges.map((entry) => {
  344. nextLocalGeneration = Math.max(nextLocalGeneration, nextMainGeneration) + 1
  345. return {
  346. entry,
  347. generation: nextLocalGeneration,
  348. persisted: observation.persisted.has(entry.header.id),
  349. }
  350. })
  351. if (hasWrites) {
  352. let began = false
  353. try {
  354. db.exec('BEGIN IMMEDIATE')
  355. began = true
  356. for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId)
  357. for (const entry of persistentChanges) {
  358. /* v8 ignore next -- observation loads every entry whose revision differs */
  359. if (entry.loaded === undefined) throw new Error(`missing loaded revision for session "${entry.header.id}"`)
  360. this._replacePersistedSession(entry.loaded, entry.revision, nextMainGeneration)
  361. }
  362. if (persistentChanges.length > 0 || persistentDeletes.length > 0) {
  363. db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration)
  364. }
  365. for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId)
  366. for (const { entry, generation, persisted } of liveReplacements) {
  367. this._replaceLiveSession(entry, generation, persisted)
  368. }
  369. db.exec('COMMIT')
  370. } catch (error: unknown) {
  371. /* v8 ignore next -- a BEGIN failure has no transaction to roll back; the common wrapper still reports it. */
  372. if (began) {
  373. /* v8 ignore next 5 -- ROLLBACK failure requires a SQLite double fault; the original failure remains actionable. */
  374. try {
  375. db.exec('ROLLBACK')
  376. } catch {
  377. // The original SQLite failure remains the actionable cause.
  378. }
  379. }
  380. throw new SessionQueryError(
  381. `session-search reconciliation failed: ${errorMessage(error)}`,
  382. 'SESSION_QUERY_INDEX_FAILED',
  383. { cause: error },
  384. )
  385. }
  386. }
  387. if (hasWrites || pointerChanged) this._globalGeneration += 1
  388. if (pointerChanged) this._persistenceEpoch += 1
  389. this._localGeneration = nextLocalGeneration
  390. this._lastPersistenceIdentity = observation.persistenceBinding.identity
  391. return observation.persistenceBinding
  392. }
  393. private async _observeStable(
  394. indexed: ReadonlyMap<SessionId, IndexedPersistedRow>,
  395. signal: AbortSignal | undefined,
  396. ): Promise<Observation> {
  397. for (let attempt = 0; attempt < STABLE_OBSERVATION_ATTEMPTS; attempt += 1) {
  398. assertNotAborted(signal)
  399. const persistenceBinding = this._persistenceBinding
  400. const persistence = persistenceBinding.service
  401. const initiallyLive = new Set(this.ctx.sessions.list().map(session => session.id))
  402. let persisted = new Map<SessionId, ObservedPersistedSession>()
  403. if (persistence !== undefined) {
  404. try {
  405. const canReuseIndexed = this._lastPersistenceIdentity === undefined
  406. || this._lastPersistenceIdentity === persistenceBinding.identity
  407. const before = await waitWithAbort(persistence.listSnapshots(), signal)
  408. persisted = materializePersistenceSnapshots(before)
  409. for (const entry of persisted.values()) {
  410. if (canReuseIndexed && indexed.get(entry.header.id)?.revision === entry.revision) continue
  411. // Skip work already shadowed by a live owner. `inspect()` is
  412. // non-mutating, so an owner attaching after this check cannot cause
  413. // crash-repair side effects; the live-membership retry below makes
  414. // the returned observation live-preferred.
  415. if (initiallyLive.has(entry.header.id) || this.ctx.sessions.get(entry.header.id) !== undefined) continue
  416. const loaded = await waitWithAbort(persistence.inspect(entry.header.id), signal)
  417. assertSessionHeadersCompatible(entry.header, loaded.meta)
  418. entry.loaded = observeSession(loaded.meta, loaded.events)
  419. }
  420. const after = materializePersistenceSnapshots(
  421. await waitWithAbort(persistence.listSnapshots(), signal),
  422. )
  423. if (!samePersistenceSnapshots(persisted, after)) continue
  424. if (this._persistenceBinding !== persistenceBinding) continue
  425. } catch (error: unknown) {
  426. if (isAbort(error) || signal?.aborted) {
  427. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', {
  428. cause: error,
  429. })
  430. }
  431. if (this._persistenceBinding !== persistenceBinding) continue
  432. if (error instanceof SessionQueryError) throw error
  433. throw new SessionQueryError(
  434. `session-search persistence observation failed: ${errorMessage(error)}`,
  435. 'SESSION_QUERY_PERSISTENCE_FAILED',
  436. { cause: error },
  437. )
  438. }
  439. }
  440. const live = new Map<SessionId, ObservedSession>()
  441. for (const session of this.ctx.sessions.list()) {
  442. const observed = observeLive(session)
  443. const durable = persisted.get(session.id)
  444. if (durable !== undefined) assertSessionHeadersCompatible(observed.header, durable.header)
  445. live.set(session.id, observed)
  446. }
  447. if (!sameSessionIds(initiallyLive, live)) continue
  448. return { persistenceBinding, persisted, live }
  449. }
  450. throw new SessionQueryError(
  451. 'session-search persistence observation did not stabilize after one retry',
  452. 'SESSION_QUERY_PERSISTENCE_FAILED',
  453. )
  454. }
  455. private _mainGeneration(): number {
  456. const row = this._requireDb().prepare(
  457. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  458. ).get() as { global_generation: number }
  459. return row.global_generation
  460. }
  461. private _deleteSession(source: 'persisted' | 'live', id: SessionId): void {
  462. const db = this._requireDb()
  463. if (source === 'persisted') {
  464. db.prepare('DELETE FROM persisted_docs WHERE session_id = ?').run(id)
  465. db.prepare('DELETE FROM persisted_sessions WHERE id = ?').run(id)
  466. } else {
  467. db.prepare('DELETE FROM temp.live_docs WHERE session_id = ?').run(id)
  468. db.prepare('DELETE FROM temp.live_sessions WHERE id = ?').run(id)
  469. }
  470. }
  471. private _replacePersistedSession(
  472. entry: ObservedSession,
  473. revision: SessionPersistenceRevision,
  474. generation: number,
  475. ): void {
  476. this._deleteSession('persisted', entry.header.id)
  477. const db = this._requireDb()
  478. db.prepare(`
  479. INSERT INTO persisted_sessions
  480. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, revision, generation)
  481. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
  482. `).run(
  483. entry.header.id,
  484. entry.header.version,
  485. entry.header.createdAt,
  486. entry.header.cwd ?? null,
  487. entry.header.parentSession ?? null,
  488. entry.header.seedLength ?? null,
  489. entry.header.delegationDepth ?? null,
  490. revision,
  491. generation,
  492. )
  493. const insert = db.prepare(`
  494. INSERT INTO persisted_docs (text, session_id, seq, type, time, surface, codepoint_length)
  495. VALUES (?, ?, ?, ?, ?, ?, ?)
  496. `)
  497. for (const document of entry.documents) {
  498. const text = sanitizeFtsText(document.text)
  499. insert.run(
  500. text,
  501. document.sessionId,
  502. document.seq,
  503. document.type,
  504. document.time,
  505. document.surface,
  506. Array.from(text).length,
  507. )
  508. }
  509. }
  510. private _replaceLiveSession(entry: ObservedSession, generation: number, persisted: boolean): void {
  511. this._deleteSession('live', entry.header.id)
  512. const db = this._requireDb()
  513. db.prepare(`
  514. INSERT INTO temp.live_sessions
  515. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, fingerprint, persisted, generation)
  516. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  517. `).run(
  518. entry.header.id,
  519. entry.header.version,
  520. entry.header.createdAt,
  521. entry.header.cwd ?? null,
  522. entry.header.parentSession ?? null,
  523. entry.header.seedLength ?? null,
  524. entry.header.delegationDepth ?? null,
  525. entry.fingerprint,
  526. persisted ? 1 : 0,
  527. generation,
  528. )
  529. const insert = db.prepare(`
  530. INSERT INTO temp.live_docs (text, session_id, seq, type, time, surface, codepoint_length)
  531. VALUES (?, ?, ?, ?, ?, ?, ?)
  532. `)
  533. for (const document of entry.documents) {
  534. const text = sanitizeFtsText(document.text)
  535. insert.run(
  536. text,
  537. document.sessionId,
  538. document.seq,
  539. document.type,
  540. document.time,
  541. document.surface,
  542. Array.from(text).length,
  543. )
  544. }
  545. }
  546. private _querySessions(
  547. request: NormalizedSessionRequest,
  548. offset: number,
  549. persistenceBinding: PersistenceBinding,
  550. ): SearchRow[] {
  551. const selected = selectedDocumentsSql()
  552. const sessionWhere = buildSessionWhere(request.sessionFilters)
  553. const eventWhere = buildEventWhere(request.eventFilters)
  554. assertFts5OuterPredicateCount(sessionWhere.predicateCount + eventWhere.predicateCount)
  555. const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ')
  556. const bindings = [
  557. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  558. ...sessionWhere.params,
  559. ...eventWhere.params,
  560. request.limit + 1,
  561. offset,
  562. ]
  563. assertPortableBindingCount(bindings.length)
  564. return this._requireDb().prepare(`
  565. ${selected.sql},
  566. filtered AS (
  567. SELECT * FROM matched ${where.length === 0 ? '' : `WHERE ${where}`}
  568. ),
  569. ranked AS (
  570. SELECT *, ROW_NUMBER() OVER (
  571. PARTITION BY session_id
  572. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  573. ) AS event_rank
  574. FROM filtered
  575. )
  576. SELECT * FROM ranked
  577. WHERE event_rank = 1
  578. ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC
  579. LIMIT ? OFFSET ?
  580. `).all(...bindings) as unknown as SearchRow[]
  581. }
  582. private _queryEvents(
  583. request: NormalizedEventRequest,
  584. offset: number,
  585. persistenceBinding: PersistenceBinding,
  586. ): SearchRow[] {
  587. const selected = selectedDocumentsSql()
  588. const eventWhere = buildEventWhere(request.filters)
  589. assertFts5OuterPredicateCount(1 + eventWhere.predicateCount)
  590. const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ')
  591. const bindings = [
  592. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  593. request.sessionId,
  594. ...eventWhere.params,
  595. request.limit + 1,
  596. offset,
  597. ]
  598. assertPortableBindingCount(bindings.length)
  599. return this._requireDb().prepare(`
  600. ${selected.sql}
  601. SELECT * FROM matched
  602. WHERE ${where}
  603. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  604. LIMIT ? OFFSET ?
  605. `).all(...bindings) as unknown as SearchRow[]
  606. }
  607. private _targetGeneration(sessionId: SessionId, persistenceBinding: PersistenceBinding): string {
  608. const db = this._requireDb()
  609. const live = db.prepare(
  610. 'SELECT generation FROM temp.live_sessions WHERE id = ?',
  611. ).get(sessionId) as { generation: number } | undefined
  612. if (live !== undefined) return `live:${live.generation}`
  613. if (persistenceBinding.service !== undefined) {
  614. const persisted = db.prepare(
  615. 'SELECT generation FROM persisted_sessions WHERE id = ?',
  616. ).get(sessionId) as { generation: number } | undefined
  617. if (persisted !== undefined) return `persisted:${this._persistenceEpoch}:${persisted.generation}`
  618. }
  619. throw new SessionQueryError(
  620. `session "${sessionId}" not found`,
  621. 'SESSION_QUERY_SESSION_NOT_FOUND',
  622. )
  623. }
  624. private _sessionHit(row: SearchRow): SessionSearchHit {
  625. return {
  626. header: rowHeader(row),
  627. live: row.live === 1,
  628. persisted: row.persisted === 1,
  629. bestMatch: this._eventHit(row),
  630. }
  631. }
  632. private _eventHit(row: SearchRow): SessionEventSearchHit {
  633. return {
  634. sessionId: row.session_id as SessionId,
  635. seq: row.seq,
  636. type: row.type as SessionEventSearchHit['type'],
  637. time: row.time,
  638. surface: row.surface as SessionEventSearchHit['surface'],
  639. snippet: makeSnippet(row.marked_text, this.config.snippetChars),
  640. }
  641. }
  642. private _requireDb(): DatabaseSync {
  643. /* v8 ignore next -- callers await `_ready`; this guards lifecycle misuse */
  644. if (this._db === undefined) throw indexClosed()
  645. return this._db
  646. }
  647. private _isClosed(): boolean {
  648. return this._closed
  649. }
  650. }
  651. function selectedDocumentsSql(): { sql: string } {
  652. return {
  653. sql: `WITH candidates AS (
  654. SELECT
  655. pd.session_id AS session_id,
  656. ps.version AS version,
  657. ps.created_at AS created_at,
  658. ps.cwd AS cwd,
  659. ps.parent_session AS parent_session,
  660. ps.seed_length AS seed_length,
  661. ps.delegation_depth AS delegation_depth,
  662. 0 AS live,
  663. 1 AS persisted,
  664. CAST(pd.seq AS INTEGER) AS seq,
  665. pd.type AS type,
  666. CAST(pd.time AS INTEGER) AS time,
  667. pd.surface AS surface,
  668. highlight(persisted_docs, 0, ?, ?) AS marked_text,
  669. CAST(pd.codepoint_length AS INTEGER) AS document_length
  670. FROM persisted_docs AS pd
  671. JOIN persisted_sessions AS ps ON ps.id = pd.session_id
  672. WHERE persisted_docs MATCH ?
  673. AND ? = 1
  674. AND NOT EXISTS (SELECT 1 FROM temp.live_sessions AS ls WHERE ls.id = pd.session_id)
  675. UNION ALL
  676. SELECT
  677. ld.session_id AS session_id,
  678. ls.version AS version,
  679. ls.created_at AS created_at,
  680. ls.cwd AS cwd,
  681. ls.parent_session AS parent_session,
  682. ls.seed_length AS seed_length,
  683. ls.delegation_depth AS delegation_depth,
  684. 1 AS live,
  685. CASE WHEN ? = 1 THEN ls.persisted ELSE 0 END AS persisted,
  686. CAST(ld.seq AS INTEGER) AS seq,
  687. ld.type AS type,
  688. CAST(ld.time AS INTEGER) AS time,
  689. ld.surface AS surface,
  690. highlight(live_docs, 0, ?, ?) AS marked_text,
  691. CAST(ld.codepoint_length AS INTEGER) AS document_length
  692. FROM temp.live_docs AS ld
  693. JOIN temp.live_sessions AS ls ON ls.id = ld.session_id
  694. WHERE live_docs MATCH ?
  695. ), matched AS (
  696. SELECT *,
  697. (
  698. length(CAST(marked_text AS BLOB))
  699. - length(CAST(replace(marked_text, ?, '') AS BLOB))
  700. ) / ? AS match_count
  701. FROM candidates
  702. )`,
  703. }
  704. }
  705. function selectedDocumentsParams(query: string, persistenceVisible: boolean): Array<string | number> {
  706. const expression = quoteFtsData(query)
  707. const visible = persistenceVisible ? 1 : 0
  708. return [
  709. FTS_HIGHLIGHT_START,
  710. FTS_HIGHLIGHT_END,
  711. expression,
  712. visible,
  713. visible,
  714. FTS_HIGHLIGHT_START,
  715. FTS_HIGHLIGHT_END,
  716. expression,
  717. FTS_HIGHLIGHT_START,
  718. Buffer.byteLength(FTS_HIGHLIGHT_START, 'utf8'),
  719. ]
  720. }
  721. function observeLive(session: Session): ObservedSession {
  722. return observeSession(session.header, session.events)
  723. }
  724. function observeSession(header: SessionHeader, events: readonly SessionEvent[]): ObservedSession {
  725. const detachedHeader = structuredClone(header)
  726. const detachedEvents = events.map(event => structuredClone(event))
  727. return {
  728. header: detachedHeader,
  729. documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents),
  730. fingerprint: createHash('sha256')
  731. .update(JSON.stringify({ header: detachedHeader, events: detachedEvents }))
  732. .digest('base64url'),
  733. }
  734. }
  735. function materializePersistenceSnapshots(
  736. snapshots: readonly SessionPersistenceSnapshot[],
  737. ): Map<SessionId, ObservedPersistedSession> {
  738. if (!isRuntimeArray(snapshots)) throw new Error('persistence snapshots must be an array')
  739. const result = new Map<SessionId, ObservedPersistedSession>()
  740. for (const snapshot of snapshots) {
  741. if (typeof snapshot.revision !== 'string') {
  742. throw new Error('persistence snapshot revision must be a string')
  743. }
  744. const header = structuredClone(snapshot.header)
  745. if (result.has(header.id)) {
  746. throw new Error(`persistence listed duplicate session "${header.id}"`)
  747. }
  748. result.set(header.id, { header, revision: snapshot.revision })
  749. }
  750. return result
  751. }
  752. function samePersistenceSnapshots(
  753. before: ReadonlyMap<SessionId, ObservedPersistedSession>,
  754. after: ReadonlyMap<SessionId, ObservedPersistedSession>,
  755. ): boolean {
  756. if (before.size !== after.size) return false
  757. for (const [id, first] of before) {
  758. const second = after.get(id)
  759. if (
  760. second === undefined
  761. || first.revision !== second.revision
  762. || !sameHeader(first.header, second.header)
  763. ) return false
  764. }
  765. return true
  766. }
  767. function sameSessionIds(
  768. before: ReadonlySet<SessionId>,
  769. after: ReadonlyMap<SessionId, ObservedSession>,
  770. ): boolean {
  771. if (before.size !== after.size) return false
  772. for (const id of before) {
  773. if (!after.has(id)) return false
  774. }
  775. return true
  776. }
  777. function sameHeader(a: SessionHeader, b: SessionHeader): boolean {
  778. return a.version === b.version
  779. && a.id === b.id
  780. && a.createdAt === b.createdAt
  781. && a.cwd === b.cwd
  782. && a.parentSession === b.parentSession
  783. && a.seedLength === b.seedLength
  784. && (a.delegationDepth ?? 0) === (b.delegationDepth ?? 0)
  785. }
  786. function rowHeader(row: SearchRow): SessionHeader {
  787. return {
  788. version: row.version,
  789. id: row.session_id as SessionId,
  790. createdAt: row.created_at,
  791. ...row.cwd === null ? {} : { cwd: row.cwd },
  792. ...row.parent_session === null ? {} : { parentSession: row.parent_session as SessionId },
  793. ...row.seed_length === null ? {} : { seedLength: row.seed_length },
  794. ...row.delegation_depth === null ? {} : { delegationDepth: row.delegation_depth },
  795. }
  796. }
  797. function page<Row, Item>(
  798. rows: readonly Row[],
  799. limit: number,
  800. convert: (row: Row) => Item,
  801. nextCursor: (offset: number) => SessionSearchCursorValue,
  802. offset: number,
  803. ): SessionSearchPage<Item> {
  804. const hasMore = rows.length > limit
  805. return {
  806. items: rows.slice(0, limit).map(convert),
  807. ...hasMore ? { nextCursor: nextCursor(offset + limit) } : {},
  808. }
  809. }
  810. function encodeCursor(payload: CursorPayload): SessionSearchCursorValue {
  811. return SessionSearchCursor(Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'))
  812. }
  813. function decodeCursor(
  814. cursor: SessionSearchCursorValue,
  815. instance: string,
  816. scope: CursorPayload['scope'],
  817. fingerprint: string,
  818. generation: string,
  819. ): number {
  820. let decoded: Partial<CursorPayload>
  821. try {
  822. decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<CursorPayload>
  823. } catch (error: unknown) {
  824. throw invalidCursor(error)
  825. }
  826. if (
  827. decoded.version !== 1
  828. || decoded.instance !== instance
  829. || decoded.scope !== scope
  830. || decoded.fingerprint !== fingerprint
  831. || !Number.isSafeInteger(decoded.offset)
  832. || decoded.offset === undefined
  833. || decoded.offset < 0
  834. ) {
  835. throw invalidCursor(new Error('cursor does not belong to this normalized request'))
  836. }
  837. if (decoded.generation !== generation) {
  838. throw new SessionQueryError(
  839. 'session-search cursor is stale because its relevant corpus changed',
  840. 'SESSION_QUERY_STALE_CURSOR',
  841. )
  842. }
  843. return decoded.offset
  844. }
  845. function invalidCursor(cause: unknown): SessionQueryError {
  846. return new SessionQueryError(
  847. 'session-search cursor is invalid',
  848. 'SESSION_QUERY_INVALID_CURSOR',
  849. { cause },
  850. )
  851. }
  852. function resolveConfig(config: Config): ResolvedConfig {
  853. const resolved: ResolvedConfig = {
  854. path: config.path,
  855. journalMode: config.journalMode ?? 'wal',
  856. defaultLimit: config.defaultLimit ?? SESSION_QUERY_SQLITE_DEFAULT_LIMIT,
  857. maxLimit: config.maxLimit ?? SESSION_QUERY_SQLITE_MAX_LIMIT,
  858. snippetChars: config.snippetChars ?? SESSION_QUERY_SQLITE_SNIPPET_CHARS,
  859. readWindowMax: config.readWindowMax ?? SESSION_QUERY_READ_WINDOW_MAX,
  860. }
  861. if (typeof resolved.path !== 'string' || resolved.path.trim().length === 0) {
  862. throw invalidConfig('path must not be blank')
  863. }
  864. assertPageLimit('defaultLimit', resolved.defaultLimit)
  865. assertPageLimit('maxLimit', resolved.maxLimit)
  866. assertPositiveInteger('snippetChars', resolved.snippetChars)
  867. if (!Number.isInteger(resolved.readWindowMax) || resolved.readWindowMax < 0) {
  868. throw invalidConfig('readWindowMax must be a non-negative integer')
  869. }
  870. if (resolved.defaultLimit > resolved.maxLimit) {
  871. throw invalidConfig('defaultLimit must be less than or equal to maxLimit')
  872. }
  873. const journalModes: readonly string[] = ['wal', 'delete', 'truncate', 'persist']
  874. if (!journalModes.includes(resolved.journalMode)) throw invalidConfig('journalMode is not supported')
  875. return resolved
  876. }
  877. function assertPositiveInteger(name: string, value: number): void {
  878. if (!Number.isInteger(value) || value < 1) throw invalidConfig(`${name} must be a positive integer`)
  879. }
  880. function assertPageLimit(name: string, value: number): void {
  881. if (!Number.isSafeInteger(value) || value < 1 || value > SQLITE_MAX_PAGE_LIMIT) {
  882. throw invalidConfig(`${name} must be an integer between 1 and ${SQLITE_MAX_PAGE_LIMIT}`)
  883. }
  884. }
  885. function invalidConfig(detail: string): SessionQueryError {
  886. return new SessionQueryError(
  887. `session-search SQLite config: ${detail}`,
  888. 'SESSION_QUERY_INVALID_CONFIG',
  889. )
  890. }
  891. function indexClosed(): SessionQueryError {
  892. return new SessionQueryError('session-search SQLite index is closed', 'SESSION_QUERY_INDEX_FAILED')
  893. }
  894. function assertNotAborted(signal: AbortSignal | undefined): void {
  895. if (signal?.aborted) {
  896. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED')
  897. }
  898. }
  899. function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> {
  900. if (signal === undefined) return promise
  901. if (signal.aborted) return Promise.reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  902. return new Promise<T>((resolve, reject) => {
  903. const onAbort = () => {
  904. reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  905. }
  906. signal.addEventListener('abort', onAbort, { once: true })
  907. promise.then(
  908. (value) => {
  909. signal.removeEventListener('abort', onAbort)
  910. resolve(value)
  911. },
  912. (error: unknown) => {
  913. signal.removeEventListener('abort', onAbort)
  914. reject(asError(error))
  915. },
  916. )
  917. })
  918. }
  919. function isAbort(error: unknown): boolean {
  920. return error instanceof SessionQueryError && error.code === 'SESSION_QUERY_ABORTED'
  921. }
  922. function asError(error: unknown): Error {
  923. return error instanceof Error
  924. ? error
  925. : new Error('session-search dependency rejected with a non-Error value', { cause: error })
  926. }
  927. function errorMessage(error: unknown): string {
  928. return error instanceof Error ? error.message : 'unknown error'
  929. }
  930. function isRuntimeArray(value: unknown): boolean {
  931. return Array.isArray(value)
  932. }
  933. export default SessionQuerySqlite