index.ts 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060
  1. /**
  2. * Concrete session-query service with SQLite FTS5 over the live-preferred corpus.
  3. *
  4. * @module @deepseek-ai/dsh-session-query-sqlite
  5. */
  6. import { createHash, randomUUID } from 'node:crypto'
  7. import type { DatabaseSync } from 'node:sqlite'
  8. import { Context, Service, type Fiber } from 'cordis'
  9. import z from 'schemastery'
  10. import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
  11. import type SessionPersistence from '@deepseek-ai/dsh-session-persistence'
  12. import type {
  13. SessionPersistenceRevision,
  14. SessionPersistenceSnapshot,
  15. } from '@deepseek-ai/dsh-session-persistence'
  16. import SessionQueryService, {
  17. SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY,
  18. SESSION_QUERY_READ_WINDOW_MAX,
  19. SessionQueryError,
  20. SessionSearchCursor,
  21. assertSessionHeadersCompatible,
  22. buildSessionEventSearchDocuments,
  23. } from '@deepseek-ai/dsh-session-query'
  24. import type {
  25. Config as SessionQueryConfig,
  26. SessionEventSearchDocument,
  27. SessionEventSearchHit,
  28. SessionEventSearchPage,
  29. SessionEventSearchRequest,
  30. SessionSearchExecContext,
  31. SessionSearchHit,
  32. SessionSearchCursor as SessionSearchCursorValue,
  33. SessionSearchPage,
  34. SessionSearchRequest,
  35. } from '@deepseek-ai/dsh-session-query'
  36. import {
  37. type JournalMode,
  38. openSearchDatabase,
  39. } from './schema.ts'
  40. import {
  41. type NormalizedEventRequest,
  42. type NormalizedSessionRequest,
  43. FTS_HIGHLIGHT_END,
  44. FTS_HIGHLIGHT_START,
  45. assertFts5OuterPredicateCount,
  46. assertPortableBindingCount,
  47. buildEventWhere,
  48. buildSessionWhere,
  49. makeSnippet,
  50. normalizeEventRequest,
  51. normalizeSessionRequest,
  52. quoteFtsData,
  53. requestFingerprint,
  54. sanitizeFtsText,
  55. SQLITE_MAX_PAGE_LIMIT,
  56. } from './query.ts'
  57. export {
  58. SESSION_QUERY_SQLITE_APPLICATION_ID,
  59. SESSION_QUERY_SQLITE_SCHEMA_VERSION,
  60. type JournalMode,
  61. } from './schema.ts'
  62. /** Default result page size. */
  63. export const SESSION_QUERY_SQLITE_DEFAULT_LIMIT = 20
  64. /** Maximum accepted result page size. */
  65. export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100
  66. /** Default maximum snippet length in Unicode code points. */
  67. export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240
  68. // One transient source change gets a retry; repeated churn fails rather than monopolizing the queue.
  69. const STABLE_OBSERVATION_ATTEMPTS = 2
  70. /** SQLite module/handle opening phase. */
  71. export type OpenAt = 'startup' | 'first-search'
  72. /** Combined session-query configuration backed by SQLite full-text search. */
  73. export interface Config extends SessionQueryConfig {
  74. /**
  75. * Dedicated derived-index path; `:memory:` is supported for ephemeral
  76. * indexes. Missing directories and database files are created owner-only on
  77. * POSIX filesystems; existing modes are preserved.
  78. */
  79. path: string
  80. /** Open the SQLite module and handle at service activation or the first search. Defaults to `startup`. */
  81. openAt?: OpenAt
  82. /** SQLite journal mode. Defaults to `wal`. */
  83. journalMode?: JournalMode
  84. /** Page size when a request omits `limit`. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 20. */
  85. defaultLimit?: number
  86. /** Largest accepted page size. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 100. */
  87. maxLimit?: number
  88. /** Maximum snippet length in Unicode code points. Defaults to 240. */
  89. snippetChars?: number
  90. /** Maximum concurrent persisted-log inspections in one inherited batch read. Defaults to 4. */
  91. persistedInspectConcurrency?: number
  92. }
  93. interface ResolvedConfig {
  94. path: string
  95. openAt: OpenAt
  96. journalMode: JournalMode
  97. defaultLimit: number
  98. maxLimit: number
  99. snippetChars: number
  100. readWindowMax: number
  101. persistedInspectConcurrency: number
  102. }
  103. interface ObservedSession {
  104. header: SessionHeader
  105. documents: SessionEventSearchDocument[]
  106. fingerprint: string
  107. }
  108. interface ObservedPersistedSession {
  109. header: SessionHeader
  110. revision: SessionPersistenceRevision
  111. loaded?: ObservedSession
  112. }
  113. interface PersistenceBinding {
  114. readonly identity: symbol
  115. readonly service?: SessionPersistence
  116. }
  117. interface Observation {
  118. persistenceBinding: PersistenceBinding
  119. persisted: Map<SessionId, ObservedPersistedSession>
  120. live: Map<SessionId, ObservedSession>
  121. }
  122. interface IndexedPersistedRow {
  123. id: string
  124. revision: string
  125. generation: number
  126. }
  127. interface IndexedLiveRow {
  128. id: string
  129. fingerprint: string
  130. persisted: number
  131. generation: number
  132. }
  133. interface SessionHeaderRow {
  134. session_id: string
  135. version: number
  136. created_at: number
  137. cwd: string | null
  138. parent_session: string | null
  139. seed_length: number | null
  140. delegation_depth: number | null
  141. }
  142. interface SearchRow extends SessionHeaderRow {
  143. live: number
  144. persisted: number
  145. seq: number
  146. type: string
  147. time: number
  148. surface: string
  149. marked_text: string
  150. match_count: number
  151. document_length: number
  152. }
  153. interface CursorPayload {
  154. version: 1
  155. instance: string
  156. scope: 'sessions' | 'events'
  157. fingerprint: string
  158. generation: string
  159. offset: number
  160. }
  161. /** Concrete SQLite owner of the combined `ctx.sessionQuery` service. */
  162. export class SessionQuerySqlite extends SessionQueryService {
  163. static override inject = ['sessions']
  164. static Config: z<Config> = z.object({
  165. path: z.string().required(),
  166. openAt: z.union(['startup', 'first-search'] as const).default('startup'),
  167. journalMode: z.union(['wal', 'delete', 'truncate', 'persist'] as const).default('wal'),
  168. defaultLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT),
  169. maxLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_MAX_LIMIT),
  170. snippetChars: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_SNIPPET_CHARS),
  171. readWindowMax: z.number().step(1).min(0).default(SESSION_QUERY_READ_WINDOW_MAX),
  172. persistedInspectConcurrency: z.number()
  173. .step(1)
  174. .min(1)
  175. .max(Number.MAX_SAFE_INTEGER)
  176. .default(SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY),
  177. })
  178. /** Validated and defaulted backend configuration. */
  179. readonly config: ResolvedConfig
  180. private readonly _instance = randomUUID()
  181. private _ready: Promise<void> | undefined
  182. private _db: DatabaseSync | undefined
  183. private _persistenceBinding: PersistenceBinding = { identity: Symbol() }
  184. private _lastPersistenceIdentity: symbol | undefined
  185. private _persistenceEpoch = 0
  186. private _globalGeneration = 0
  187. private _localGeneration = 0
  188. private _tail: Promise<void> = Promise.resolve()
  189. private _closed = false
  190. private _closePromise: Promise<void> | undefined
  191. private readonly _optionalPersistenceFiber: Fiber
  192. constructor(ctx: Context, config: Config) {
  193. // The assignment expression resolves before the base constructor can
  194. // register `ctx.sessionQuery`; keep that same validated value afterward.
  195. super(ctx, config = resolveConfig(config))
  196. this.config = config as ResolvedConfig
  197. this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => {
  198. const service = childCtx.sessionPersistence
  199. const binding = { identity: Symbol(), service }
  200. this._persistenceBinding = binding
  201. childCtx.effect(() => () => {
  202. /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */
  203. if (this._persistenceBinding !== binding) return
  204. this._persistenceBinding = { identity: Symbol() }
  205. }, 'sessionQuerySqlite.persistenceBinding')
  206. })
  207. ctx.effect(() => {
  208. return () => this._optionalPersistenceFiber.dispose()
  209. }, 'sessionQuerySqlite.optionalPersistence')
  210. ctx.effect(() => async () => this.close(), 'sessionQuerySqlite.close')
  211. }
  212. /** Open eagerly only when activation owns the configured readiness boundary. */
  213. protected async [Service.init](): Promise<void> {
  214. if (this.config.openAt === 'startup') await this._ensureReady(undefined)
  215. }
  216. override async searchSessions(
  217. request: SessionSearchRequest,
  218. exec?: SessionSearchExecContext,
  219. ): Promise<SessionSearchPage<SessionSearchHit>> {
  220. const normalized = normalizeSessionRequest(request, this.config)
  221. const signal = exec?.signal
  222. return this._serialized(signal, async () => {
  223. await this._ensureReady(signal)
  224. const persistenceBinding = await this._reconcile(signal)
  225. assertNotAborted(signal)
  226. const generation = String(this._globalGeneration)
  227. const fingerprint = requestFingerprint(normalized)
  228. const offset = normalized.cursor === undefined
  229. ? 0
  230. : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation)
  231. const rows = this._querySessions(normalized, offset, persistenceBinding)
  232. return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({
  233. version: 1,
  234. instance: this._instance,
  235. scope: 'sessions',
  236. fingerprint,
  237. generation,
  238. offset: cursorOffset,
  239. }), offset)
  240. })
  241. }
  242. override async searchEvents(
  243. request: SessionEventSearchRequest,
  244. exec?: SessionSearchExecContext,
  245. ): Promise<SessionEventSearchPage> {
  246. const normalized = normalizeEventRequest(request, this.config)
  247. const signal = exec?.signal
  248. return this._serialized(signal, async () => {
  249. await this._ensureReady(signal)
  250. const persistenceBinding = await this._reconcile(signal)
  251. assertNotAborted(signal)
  252. const target = this._targetObservation(normalized.sessionId, persistenceBinding)
  253. const fingerprint = requestFingerprint(normalized)
  254. const offset = normalized.cursor === undefined
  255. ? 0
  256. : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, target.generation)
  257. const rows = this._queryEvents(normalized, offset, persistenceBinding)
  258. return {
  259. session: target.header,
  260. ...page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({
  261. version: 1,
  262. instance: this._instance,
  263. scope: 'events',
  264. fingerprint,
  265. generation: target.generation,
  266. offset: cursorOffset,
  267. }), offset),
  268. }
  269. })
  270. }
  271. /** Close the database after every accepted operation reaches quiescence. */
  272. close(): Promise<void> {
  273. this._closePromise ??= this._close()
  274. return this._closePromise
  275. }
  276. private async _close(): Promise<void> {
  277. this._closed = true
  278. await this._tail
  279. if (this._ready !== undefined) {
  280. try {
  281. await this._ready
  282. } catch {
  283. // Opening already closed a partially-created handle; disposal only waits.
  284. }
  285. }
  286. this._db?.close()
  287. this._db = undefined
  288. }
  289. private async _open(): Promise<void> {
  290. this._db = await openSearchDatabase(this.config.path, this.config.journalMode)
  291. const state = this._db.prepare(
  292. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  293. ).get() as { global_generation: number }
  294. this._globalGeneration = state.global_generation
  295. this._localGeneration = state.global_generation
  296. }
  297. private async _ensureReady(signal: AbortSignal | undefined): Promise<void> {
  298. this._ready ??= this._open()
  299. try {
  300. await waitWithAbort(this._ready, signal)
  301. } catch (error: unknown) {
  302. if (isAbort(error)) throw error
  303. throw new SessionQueryError(
  304. `session-search SQLite index failed to open: ${errorMessage(error)}`,
  305. 'SESSION_QUERY_INDEX_FAILED',
  306. { cause: error },
  307. )
  308. }
  309. }
  310. private async _serialized<T>(signal: AbortSignal | undefined, operation: () => Promise<T>): Promise<T> {
  311. if (this._isClosed()) throw indexClosed()
  312. let release!: () => void
  313. const gate = new Promise<void>((resolve) => { release = resolve })
  314. const prior = this._tail
  315. this._tail = prior.then(() => gate)
  316. try {
  317. await waitWithAbort(prior, signal)
  318. } catch (error: unknown) {
  319. release()
  320. throw error
  321. }
  322. if (this._isClosed()) {
  323. release()
  324. throw indexClosed()
  325. }
  326. try {
  327. assertNotAborted(signal)
  328. return await operation()
  329. } finally {
  330. release()
  331. }
  332. }
  333. private async _reconcile(signal: AbortSignal | undefined): Promise<PersistenceBinding> {
  334. assertNotAborted(signal)
  335. const db = this._requireDb()
  336. const persistedRows = db.prepare(
  337. 'SELECT id, revision, generation FROM persisted_sessions',
  338. ).all() as unknown as IndexedPersistedRow[]
  339. const liveRows = db.prepare(
  340. 'SELECT id, fingerprint, persisted, generation FROM temp.live_sessions',
  341. ).all() as unknown as IndexedLiveRow[]
  342. const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row]))
  343. const liveById = new Map(liveRows.map(row => [row.id as SessionId, row]))
  344. const observation = await this._observeStable(persistedById, signal)
  345. assertNotAborted(signal)
  346. const persistentChanges = observation.persistenceBinding.service === undefined
  347. ? []
  348. : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined)
  349. const persistentDeletes = observation.persistenceBinding.service === undefined
  350. ? []
  351. : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId))
  352. const liveChanges = [...observation.live.values()].filter((entry) => {
  353. const indexed = liveById.get(entry.header.id)
  354. const persisted = observation.persisted.has(entry.header.id) ? 1 : 0
  355. return indexed?.fingerprint !== entry.fingerprint || indexed.persisted !== persisted
  356. })
  357. const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId))
  358. const pointerChanged = this._lastPersistenceIdentity !== undefined
  359. && this._lastPersistenceIdentity !== observation.persistenceBinding.identity
  360. const hasWrites = persistentChanges.length > 0
  361. || persistentDeletes.length > 0
  362. || liveChanges.length > 0
  363. || liveDeletes.length > 0
  364. let nextMainGeneration = this._mainGeneration()
  365. let nextLocalGeneration = this._localGeneration
  366. if (persistentChanges.length > 0 || persistentDeletes.length > 0) nextMainGeneration += 1
  367. const liveReplacements = liveChanges.map((entry) => {
  368. nextLocalGeneration = Math.max(nextLocalGeneration, nextMainGeneration) + 1
  369. return {
  370. entry,
  371. generation: nextLocalGeneration,
  372. persisted: observation.persisted.has(entry.header.id),
  373. }
  374. })
  375. if (hasWrites) {
  376. let began = false
  377. try {
  378. db.exec('BEGIN IMMEDIATE')
  379. began = true
  380. for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId)
  381. for (const entry of persistentChanges) {
  382. /* v8 ignore next -- observation loads every entry whose revision differs */
  383. if (entry.loaded === undefined) throw new Error(`missing loaded revision for session "${entry.header.id}"`)
  384. this._replacePersistedSession(entry.loaded, entry.revision, nextMainGeneration)
  385. }
  386. if (persistentChanges.length > 0 || persistentDeletes.length > 0) {
  387. db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration)
  388. }
  389. for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId)
  390. for (const { entry, generation, persisted } of liveReplacements) {
  391. this._replaceLiveSession(entry, generation, persisted)
  392. }
  393. db.exec('COMMIT')
  394. } catch (error: unknown) {
  395. /* v8 ignore next -- a BEGIN failure has no transaction to roll back; the common wrapper still reports it. */
  396. if (began) {
  397. /* v8 ignore next 5 -- ROLLBACK failure requires a SQLite double fault; the original failure remains actionable. */
  398. try {
  399. db.exec('ROLLBACK')
  400. } catch {
  401. // The original SQLite failure remains the actionable cause.
  402. }
  403. }
  404. throw new SessionQueryError(
  405. `session-search reconciliation failed: ${errorMessage(error)}`,
  406. 'SESSION_QUERY_INDEX_FAILED',
  407. { cause: error },
  408. )
  409. }
  410. }
  411. if (hasWrites || pointerChanged) this._globalGeneration += 1
  412. if (pointerChanged) this._persistenceEpoch += 1
  413. this._localGeneration = nextLocalGeneration
  414. this._lastPersistenceIdentity = observation.persistenceBinding.identity
  415. return observation.persistenceBinding
  416. }
  417. private async _observeStable(
  418. indexed: ReadonlyMap<SessionId, IndexedPersistedRow>,
  419. signal: AbortSignal | undefined,
  420. ): Promise<Observation> {
  421. for (let attempt = 0; attempt < STABLE_OBSERVATION_ATTEMPTS; attempt += 1) {
  422. assertNotAborted(signal)
  423. const persistenceBinding = this._persistenceBinding
  424. const persistence = persistenceBinding.service
  425. const initiallyLive = new Set(this.ctx.sessions.list().map(session => session.id))
  426. let persisted = new Map<SessionId, ObservedPersistedSession>()
  427. if (persistence !== undefined) {
  428. try {
  429. const canReuseIndexed = this._lastPersistenceIdentity === undefined
  430. || this._lastPersistenceIdentity === persistenceBinding.identity
  431. const before = await persistence.listSnapshots(signal)
  432. assertNotAborted(signal)
  433. persisted = materializePersistenceSnapshots(before)
  434. for (const entry of persisted.values()) {
  435. if (canReuseIndexed && indexed.get(entry.header.id)?.revision === entry.revision) continue
  436. // Skip work already shadowed by a live owner. `inspect()` is
  437. // non-mutating, so an owner attaching after this check cannot cause
  438. // crash-repair side effects; the live-membership retry below makes
  439. // the returned observation live-preferred.
  440. if (initiallyLive.has(entry.header.id) || this.ctx.sessions.get(entry.header.id) !== undefined) continue
  441. assertNotAborted(signal)
  442. const loaded = await persistence.inspect(entry.header.id, signal)
  443. assertNotAborted(signal)
  444. assertSessionHeadersCompatible(entry.header, loaded.meta)
  445. entry.loaded = observeSession(loaded.meta, loaded.events)
  446. }
  447. assertNotAborted(signal)
  448. const afterSnapshots = await persistence.listSnapshots(signal)
  449. assertNotAborted(signal)
  450. const after = materializePersistenceSnapshots(afterSnapshots)
  451. if (!samePersistenceSnapshots(persisted, after)) continue
  452. if (this._persistenceBinding !== persistenceBinding) continue
  453. } catch (error: unknown) {
  454. if (isAbort(error) || signal?.aborted) {
  455. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', {
  456. cause: error,
  457. })
  458. }
  459. if (this._persistenceBinding !== persistenceBinding) continue
  460. if (error instanceof SessionQueryError) throw error
  461. throw new SessionQueryError(
  462. `session-search persistence observation failed: ${errorMessage(error)}`,
  463. 'SESSION_QUERY_PERSISTENCE_FAILED',
  464. { cause: error },
  465. )
  466. }
  467. }
  468. const live = new Map<SessionId, ObservedSession>()
  469. for (const session of this.ctx.sessions.list()) {
  470. const observed = observeLive(session)
  471. const durable = persisted.get(session.id)
  472. if (durable !== undefined) assertSessionHeadersCompatible(observed.header, durable.header)
  473. live.set(session.id, observed)
  474. }
  475. if (!sameSessionIds(initiallyLive, live)) continue
  476. return { persistenceBinding, persisted, live }
  477. }
  478. throw new SessionQueryError(
  479. 'session-search persistence observation did not stabilize after one retry',
  480. 'SESSION_QUERY_PERSISTENCE_FAILED',
  481. )
  482. }
  483. private _mainGeneration(): number {
  484. const row = this._requireDb().prepare(
  485. 'SELECT global_generation FROM search_state WHERE singleton = 1',
  486. ).get() as { global_generation: number }
  487. return row.global_generation
  488. }
  489. private _deleteSession(source: 'persisted' | 'live', id: SessionId): void {
  490. const db = this._requireDb()
  491. if (source === 'persisted') {
  492. db.prepare('DELETE FROM persisted_docs WHERE session_id = ?').run(id)
  493. db.prepare('DELETE FROM persisted_sessions WHERE id = ?').run(id)
  494. } else {
  495. db.prepare('DELETE FROM temp.live_docs WHERE session_id = ?').run(id)
  496. db.prepare('DELETE FROM temp.live_sessions WHERE id = ?').run(id)
  497. }
  498. }
  499. private _replacePersistedSession(
  500. entry: ObservedSession,
  501. revision: SessionPersistenceRevision,
  502. generation: number,
  503. ): void {
  504. this._deleteSession('persisted', entry.header.id)
  505. const db = this._requireDb()
  506. db.prepare(`
  507. INSERT INTO persisted_sessions
  508. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, revision, generation)
  509. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
  510. `).run(
  511. entry.header.id,
  512. entry.header.version,
  513. entry.header.createdAt,
  514. entry.header.cwd ?? null,
  515. entry.header.parentSession ?? null,
  516. entry.header.seedLength ?? null,
  517. entry.header.delegationDepth ?? null,
  518. revision,
  519. generation,
  520. )
  521. const insert = db.prepare(`
  522. INSERT INTO persisted_docs (text, session_id, seq, type, time, surface, codepoint_length)
  523. VALUES (?, ?, ?, ?, ?, ?, ?)
  524. `)
  525. for (const document of entry.documents) {
  526. const text = sanitizeFtsText(document.text)
  527. insert.run(
  528. text,
  529. document.sessionId,
  530. document.seq,
  531. document.type,
  532. document.time,
  533. document.surface,
  534. Array.from(text).length,
  535. )
  536. }
  537. }
  538. private _replaceLiveSession(entry: ObservedSession, generation: number, persisted: boolean): void {
  539. this._deleteSession('live', entry.header.id)
  540. const db = this._requireDb()
  541. db.prepare(`
  542. INSERT INTO temp.live_sessions
  543. (id, version, created_at, cwd, parent_session, seed_length, delegation_depth, fingerprint, persisted, generation)
  544. VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  545. `).run(
  546. entry.header.id,
  547. entry.header.version,
  548. entry.header.createdAt,
  549. entry.header.cwd ?? null,
  550. entry.header.parentSession ?? null,
  551. entry.header.seedLength ?? null,
  552. entry.header.delegationDepth ?? null,
  553. entry.fingerprint,
  554. persisted ? 1 : 0,
  555. generation,
  556. )
  557. const insert = db.prepare(`
  558. INSERT INTO temp.live_docs (text, session_id, seq, type, time, surface, codepoint_length)
  559. VALUES (?, ?, ?, ?, ?, ?, ?)
  560. `)
  561. for (const document of entry.documents) {
  562. const text = sanitizeFtsText(document.text)
  563. insert.run(
  564. text,
  565. document.sessionId,
  566. document.seq,
  567. document.type,
  568. document.time,
  569. document.surface,
  570. Array.from(text).length,
  571. )
  572. }
  573. }
  574. private _querySessions(
  575. request: NormalizedSessionRequest,
  576. offset: number,
  577. persistenceBinding: PersistenceBinding,
  578. ): SearchRow[] {
  579. const selected = selectedDocumentsSql()
  580. const sessionWhere = buildSessionWhere(request.sessionFilters)
  581. const eventWhere = buildEventWhere(request.eventFilters)
  582. assertFts5OuterPredicateCount(sessionWhere.predicateCount + eventWhere.predicateCount)
  583. const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ')
  584. const bindings = [
  585. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  586. ...sessionWhere.params,
  587. ...eventWhere.params,
  588. request.limit + 1,
  589. offset,
  590. ]
  591. assertPortableBindingCount(bindings.length)
  592. // The browser fixture mirrors these rank keys in
  593. // `packages/client/connection/src/client/fixture.ts`; update both together.
  594. return this._requireDb().prepare(`
  595. ${selected.sql},
  596. filtered AS (
  597. SELECT * FROM matched ${where.length === 0 ? '' : `WHERE ${where}`}
  598. ),
  599. ranked AS (
  600. SELECT *, ROW_NUMBER() OVER (
  601. PARTITION BY session_id
  602. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  603. ) AS event_rank
  604. FROM filtered
  605. )
  606. SELECT * FROM ranked
  607. WHERE event_rank = 1
  608. ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC
  609. LIMIT ? OFFSET ?
  610. `).all(...bindings) as unknown as SearchRow[]
  611. }
  612. private _queryEvents(
  613. request: NormalizedEventRequest,
  614. offset: number,
  615. persistenceBinding: PersistenceBinding,
  616. ): SearchRow[] {
  617. const selected = selectedDocumentsSql()
  618. const eventWhere = buildEventWhere(request.filters)
  619. assertFts5OuterPredicateCount(1 + eventWhere.predicateCount)
  620. const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ')
  621. const bindings = [
  622. ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined),
  623. request.sessionId,
  624. ...eventWhere.params,
  625. request.limit + 1,
  626. offset,
  627. ]
  628. assertPortableBindingCount(bindings.length)
  629. return this._requireDb().prepare(`
  630. ${selected.sql}
  631. SELECT * FROM matched
  632. WHERE ${where}
  633. ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC
  634. LIMIT ? OFFSET ?
  635. `).all(...bindings) as unknown as SearchRow[]
  636. }
  637. private _targetObservation(
  638. sessionId: SessionId,
  639. persistenceBinding: PersistenceBinding,
  640. ): { header: SessionHeader; generation: string } {
  641. const db = this._requireDb()
  642. const live = db.prepare(
  643. `SELECT
  644. id AS session_id, version, created_at, cwd, parent_session, seed_length, delegation_depth, generation
  645. FROM temp.live_sessions
  646. WHERE id = ?`,
  647. ).get(sessionId) as (SessionHeaderRow & { generation: number }) | undefined
  648. if (live !== undefined) {
  649. return { header: rowHeader(live), generation: `live:${live.generation}` }
  650. }
  651. if (persistenceBinding.service !== undefined) {
  652. const persisted = db.prepare(
  653. `SELECT
  654. id AS session_id, version, created_at, cwd, parent_session, seed_length, delegation_depth, generation
  655. FROM persisted_sessions
  656. WHERE id = ?`,
  657. ).get(sessionId) as (SessionHeaderRow & { generation: number }) | undefined
  658. if (persisted !== undefined) {
  659. return {
  660. header: rowHeader(persisted),
  661. generation: `persisted:${this._persistenceEpoch}:${persisted.generation}`,
  662. }
  663. }
  664. }
  665. throw new SessionQueryError(
  666. `session "${sessionId}" not found`,
  667. 'SESSION_QUERY_SESSION_NOT_FOUND',
  668. )
  669. }
  670. private _sessionHit(row: SearchRow): SessionSearchHit {
  671. return {
  672. header: rowHeader(row),
  673. live: row.live === 1,
  674. persisted: row.persisted === 1,
  675. bestMatch: this._eventHit(row),
  676. }
  677. }
  678. private _eventHit(row: SearchRow): SessionEventSearchHit {
  679. return {
  680. sessionId: row.session_id as SessionId,
  681. seq: row.seq,
  682. type: row.type as SessionEventSearchHit['type'],
  683. time: row.time,
  684. surface: row.surface as SessionEventSearchHit['surface'],
  685. snippet: makeSnippet(row.marked_text, this.config.snippetChars),
  686. }
  687. }
  688. private _requireDb(): DatabaseSync {
  689. /* v8 ignore next -- callers await `_ready`; this guards lifecycle misuse */
  690. if (this._db === undefined) throw indexClosed()
  691. return this._db
  692. }
  693. private _isClosed(): boolean {
  694. return this._closed
  695. }
  696. }
  697. function selectedDocumentsSql(): { sql: string } {
  698. return {
  699. sql: `WITH candidates AS (
  700. SELECT
  701. pd.session_id AS session_id,
  702. ps.version AS version,
  703. ps.created_at AS created_at,
  704. ps.cwd AS cwd,
  705. ps.parent_session AS parent_session,
  706. ps.seed_length AS seed_length,
  707. ps.delegation_depth AS delegation_depth,
  708. 0 AS live,
  709. 1 AS persisted,
  710. CAST(pd.seq AS INTEGER) AS seq,
  711. pd.type AS type,
  712. CAST(pd.time AS INTEGER) AS time,
  713. pd.surface AS surface,
  714. highlight(persisted_docs, 0, ?, ?) AS marked_text,
  715. CAST(pd.codepoint_length AS INTEGER) AS document_length
  716. FROM persisted_docs AS pd
  717. JOIN persisted_sessions AS ps ON ps.id = pd.session_id
  718. WHERE persisted_docs MATCH ?
  719. AND ? = 1
  720. AND NOT EXISTS (SELECT 1 FROM temp.live_sessions AS ls WHERE ls.id = pd.session_id)
  721. UNION ALL
  722. SELECT
  723. ld.session_id AS session_id,
  724. ls.version AS version,
  725. ls.created_at AS created_at,
  726. ls.cwd AS cwd,
  727. ls.parent_session AS parent_session,
  728. ls.seed_length AS seed_length,
  729. ls.delegation_depth AS delegation_depth,
  730. 1 AS live,
  731. CASE WHEN ? = 1 THEN ls.persisted ELSE 0 END AS persisted,
  732. CAST(ld.seq AS INTEGER) AS seq,
  733. ld.type AS type,
  734. CAST(ld.time AS INTEGER) AS time,
  735. ld.surface AS surface,
  736. highlight(live_docs, 0, ?, ?) AS marked_text,
  737. CAST(ld.codepoint_length AS INTEGER) AS document_length
  738. FROM temp.live_docs AS ld
  739. JOIN temp.live_sessions AS ls ON ls.id = ld.session_id
  740. WHERE live_docs MATCH ?
  741. ), matched AS (
  742. SELECT *,
  743. (
  744. length(CAST(marked_text AS BLOB))
  745. - length(CAST(replace(marked_text, ?, '') AS BLOB))
  746. ) / ? AS match_count
  747. FROM candidates
  748. )`,
  749. }
  750. }
  751. function selectedDocumentsParams(query: string, persistenceVisible: boolean): Array<string | number> {
  752. const expression = quoteFtsData(query)
  753. const visible = persistenceVisible ? 1 : 0
  754. return [
  755. FTS_HIGHLIGHT_START,
  756. FTS_HIGHLIGHT_END,
  757. expression,
  758. visible,
  759. visible,
  760. FTS_HIGHLIGHT_START,
  761. FTS_HIGHLIGHT_END,
  762. expression,
  763. FTS_HIGHLIGHT_START,
  764. Buffer.byteLength(FTS_HIGHLIGHT_START, 'utf8'),
  765. ]
  766. }
  767. function observeLive(session: Session): ObservedSession {
  768. return observeSession(session.header, session.events)
  769. }
  770. function observeSession(header: SessionHeader, events: readonly SessionEvent[]): ObservedSession {
  771. const detachedHeader = structuredClone(header)
  772. const detachedEvents = events.map(event => structuredClone(event))
  773. return {
  774. header: detachedHeader,
  775. documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents),
  776. fingerprint: createHash('sha256')
  777. .update(JSON.stringify({ header: detachedHeader, events: detachedEvents }))
  778. .digest('base64url'),
  779. }
  780. }
  781. function materializePersistenceSnapshots(
  782. snapshots: readonly SessionPersistenceSnapshot[],
  783. ): Map<SessionId, ObservedPersistedSession> {
  784. if (!isRuntimeArray(snapshots)) throw new Error('persistence snapshots must be an array')
  785. const result = new Map<SessionId, ObservedPersistedSession>()
  786. for (const snapshot of snapshots) {
  787. if (typeof snapshot.revision !== 'string') {
  788. throw new Error('persistence snapshot revision must be a string')
  789. }
  790. const header = structuredClone(snapshot.header)
  791. if (result.has(header.id)) {
  792. throw new Error(`persistence listed duplicate session "${header.id}"`)
  793. }
  794. result.set(header.id, { header, revision: snapshot.revision })
  795. }
  796. return result
  797. }
  798. function samePersistenceSnapshots(
  799. before: ReadonlyMap<SessionId, ObservedPersistedSession>,
  800. after: ReadonlyMap<SessionId, ObservedPersistedSession>,
  801. ): boolean {
  802. if (before.size !== after.size) return false
  803. for (const [id, first] of before) {
  804. const second = after.get(id)
  805. if (
  806. second === undefined
  807. || first.revision !== second.revision
  808. || !sameHeader(first.header, second.header)
  809. ) return false
  810. }
  811. return true
  812. }
  813. function sameSessionIds(
  814. before: ReadonlySet<SessionId>,
  815. after: ReadonlyMap<SessionId, ObservedSession>,
  816. ): boolean {
  817. if (before.size !== after.size) return false
  818. for (const id of before) {
  819. if (!after.has(id)) return false
  820. }
  821. return true
  822. }
  823. function sameHeader(a: SessionHeader, b: SessionHeader): boolean {
  824. return a.version === b.version
  825. && a.id === b.id
  826. && a.createdAt === b.createdAt
  827. && a.cwd === b.cwd
  828. && a.parentSession === b.parentSession
  829. && a.seedLength === b.seedLength
  830. && (a.delegationDepth ?? 0) === (b.delegationDepth ?? 0)
  831. }
  832. function rowHeader(row: SessionHeaderRow): SessionHeader {
  833. return {
  834. version: row.version,
  835. id: row.session_id as SessionId,
  836. createdAt: row.created_at,
  837. ...row.cwd === null ? {} : { cwd: row.cwd },
  838. ...row.parent_session === null ? {} : { parentSession: row.parent_session as SessionId },
  839. ...row.seed_length === null ? {} : { seedLength: row.seed_length },
  840. ...row.delegation_depth === null ? {} : { delegationDepth: row.delegation_depth },
  841. }
  842. }
  843. function page<Row, Item>(
  844. rows: readonly Row[],
  845. limit: number,
  846. convert: (row: Row) => Item,
  847. nextCursor: (offset: number) => SessionSearchCursorValue,
  848. offset: number,
  849. ): SessionSearchPage<Item> {
  850. const hasMore = rows.length > limit
  851. return {
  852. items: rows.slice(0, limit).map(convert),
  853. ...hasMore ? { nextCursor: nextCursor(offset + limit) } : {},
  854. }
  855. }
  856. function encodeCursor(payload: CursorPayload): SessionSearchCursorValue {
  857. return SessionSearchCursor(Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'))
  858. }
  859. function decodeCursor(
  860. cursor: SessionSearchCursorValue,
  861. instance: string,
  862. scope: CursorPayload['scope'],
  863. fingerprint: string,
  864. generation: string,
  865. ): number {
  866. let decoded: Partial<CursorPayload>
  867. try {
  868. decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<CursorPayload>
  869. } catch (error: unknown) {
  870. throw invalidCursor(error)
  871. }
  872. if (
  873. decoded.version !== 1
  874. || decoded.instance !== instance
  875. || decoded.scope !== scope
  876. || decoded.fingerprint !== fingerprint
  877. || !Number.isSafeInteger(decoded.offset)
  878. || decoded.offset === undefined
  879. || decoded.offset < 0
  880. ) {
  881. throw invalidCursor(new Error('cursor does not belong to this normalized request'))
  882. }
  883. if (decoded.generation !== generation) {
  884. throw new SessionQueryError(
  885. 'session-search cursor is stale because its relevant corpus changed',
  886. 'SESSION_QUERY_STALE_CURSOR',
  887. )
  888. }
  889. return decoded.offset
  890. }
  891. function invalidCursor(cause: unknown): SessionQueryError {
  892. return new SessionQueryError(
  893. 'session-search cursor is invalid',
  894. 'SESSION_QUERY_INVALID_CURSOR',
  895. { cause },
  896. )
  897. }
  898. function resolveConfig(config: Config): ResolvedConfig {
  899. const resolved: ResolvedConfig = {
  900. path: config.path,
  901. openAt: config.openAt ?? 'startup',
  902. journalMode: config.journalMode ?? 'wal',
  903. defaultLimit: config.defaultLimit ?? SESSION_QUERY_SQLITE_DEFAULT_LIMIT,
  904. maxLimit: config.maxLimit ?? SESSION_QUERY_SQLITE_MAX_LIMIT,
  905. snippetChars: config.snippetChars ?? SESSION_QUERY_SQLITE_SNIPPET_CHARS,
  906. readWindowMax: config.readWindowMax ?? SESSION_QUERY_READ_WINDOW_MAX,
  907. persistedInspectConcurrency: config.persistedInspectConcurrency
  908. ?? SESSION_QUERY_DEFAULT_PERSISTED_INSPECT_CONCURRENCY,
  909. }
  910. if (typeof resolved.path !== 'string' || resolved.path.trim().length === 0) {
  911. throw invalidConfig('path must not be blank')
  912. }
  913. const openPhases: readonly string[] = ['startup', 'first-search']
  914. if (!openPhases.includes(resolved.openAt)) throw invalidConfig('openAt is not supported')
  915. assertPageLimit('defaultLimit', resolved.defaultLimit)
  916. assertPageLimit('maxLimit', resolved.maxLimit)
  917. assertPositiveInteger('snippetChars', resolved.snippetChars)
  918. if (!Number.isInteger(resolved.readWindowMax) || resolved.readWindowMax < 0) {
  919. throw invalidConfig('readWindowMax must be a non-negative integer')
  920. }
  921. if (
  922. !Number.isSafeInteger(resolved.persistedInspectConcurrency)
  923. || resolved.persistedInspectConcurrency < 1
  924. ) {
  925. throw invalidConfig('persistedInspectConcurrency must be a positive safe integer')
  926. }
  927. if (resolved.defaultLimit > resolved.maxLimit) {
  928. throw invalidConfig('defaultLimit must be less than or equal to maxLimit')
  929. }
  930. const journalModes: readonly string[] = ['wal', 'delete', 'truncate', 'persist']
  931. if (!journalModes.includes(resolved.journalMode)) throw invalidConfig('journalMode is not supported')
  932. return resolved
  933. }
  934. function assertPositiveInteger(name: string, value: number): void {
  935. if (!Number.isInteger(value) || value < 1) throw invalidConfig(`${name} must be a positive integer`)
  936. }
  937. function assertPageLimit(name: string, value: number): void {
  938. if (!Number.isSafeInteger(value) || value < 1 || value > SQLITE_MAX_PAGE_LIMIT) {
  939. throw invalidConfig(`${name} must be an integer between 1 and ${SQLITE_MAX_PAGE_LIMIT}`)
  940. }
  941. }
  942. function invalidConfig(detail: string): SessionQueryError {
  943. return new SessionQueryError(
  944. `session-search SQLite config: ${detail}`,
  945. 'SESSION_QUERY_INVALID_CONFIG',
  946. )
  947. }
  948. function indexClosed(): SessionQueryError {
  949. return new SessionQueryError('session-search SQLite index is closed', 'SESSION_QUERY_INDEX_FAILED')
  950. }
  951. function assertNotAborted(signal: AbortSignal | undefined): void {
  952. if (signal?.aborted) {
  953. throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED')
  954. }
  955. }
  956. function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> {
  957. if (signal === undefined) return promise
  958. if (signal.aborted) return Promise.reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  959. return new Promise<T>((resolve, reject) => {
  960. const onAbort = () => {
  961. reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED'))
  962. }
  963. signal.addEventListener('abort', onAbort, { once: true })
  964. promise.then(
  965. (value) => {
  966. signal.removeEventListener('abort', onAbort)
  967. resolve(value)
  968. },
  969. (error: unknown) => {
  970. signal.removeEventListener('abort', onAbort)
  971. reject(asError(error))
  972. },
  973. )
  974. })
  975. }
  976. function isAbort(error: unknown): boolean {
  977. return error instanceof SessionQueryError && error.code === 'SESSION_QUERY_ABORTED'
  978. }
  979. function asError(error: unknown): Error {
  980. return error instanceof Error
  981. ? error
  982. : new Error('session-search dependency rejected with a non-Error value', { cause: error })
  983. }
  984. function errorMessage(error: unknown): string {
  985. return error instanceof Error ? error.message : 'unknown error'
  986. }
  987. function isRuntimeArray(value: unknown): boolean {
  988. return Array.isArray(value)
  989. }
  990. export default SessionQuerySqlite