|
|
@@ -33,11 +33,21 @@ const testsDir = dirOf(import.meta.url)
|
|
|
const snapshotsDir = join(testsDir, 'snapshots')
|
|
|
const liveConfig = join(testsDir, '..', 'cordis.yml')
|
|
|
const replayConfig = join(testsDir, '..', 'cordis.snapshot.yml')
|
|
|
-const persistentToolsLiveConfig = join(testsDir, '..', 'persistent-tools.cordis.yml')
|
|
|
-const persistentToolsReplayConfig = join(testsDir, '..', 'persistent-tools.snapshot.cordis.yml')
|
|
|
+const minimalLiveConfig = join(testsDir, '..', 'minimal.cordis.yml')
|
|
|
+const minimalReplayConfig = join(testsDir, '..', 'minimal.snapshot.cordis.yml')
|
|
|
const runtimeBin = fileURLToPath(new URL('../../../packages/examples/jsonrpc-demo/src/bin.ts', import.meta.url))
|
|
|
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
|
|
|
|
|
|
+const MINIMAL_SYSTEM_PROMPT = 'You are a helpful software engineer assistant.'
|
|
|
+const MINIMAL_BASH_DESCRIPTION = `Run commands in a bash shell
|
|
|
+* When invoking this tool, the contents of the "command" parameter does NOT need to be XML-escaped.
|
|
|
+* You don't have access to the internet via this tool.
|
|
|
+* You do have access to a mirror of common linux and python packages via apt and pip.
|
|
|
+* State is persistent across command calls and discussions with the user.
|
|
|
+* To inspect a particular line range of a file, e.g. lines 10-25, try 'sed -n 10,25p /path/to/the/file'.
|
|
|
+* Please avoid commands that may produce a very large amount of output.
|
|
|
+* Please run long lived commands in the background, e.g. 'sleep 10 &' or start a server in the background.`
|
|
|
+
|
|
|
const mode = process.env.DSH_SNAPSHOT ?? 'replay'
|
|
|
const recording = mode === 'record'
|
|
|
const refreshing = mode === 'refresh'
|
|
|
@@ -61,6 +71,10 @@ interface SdkScenario {
|
|
|
expectedFiles?: Readonly<Record<string, string>>
|
|
|
/** Assembled model-facing tool names and required argument keys. */
|
|
|
expectedTools?: Readonly<Record<string, readonly string[]>>
|
|
|
+ /** Exact assembled system prompt for the root request. */
|
|
|
+ expectedSystem?: string
|
|
|
+ /** Exact model-facing descriptions for selected tools. */
|
|
|
+ expectedToolDescriptions?: Readonly<Record<string, string>>
|
|
|
/** Stable policy-context clauses the real assembled request must include or omit. */
|
|
|
policyContext?: { includes: readonly string[]; excludes: readonly string[] }
|
|
|
}
|
|
|
@@ -89,9 +103,11 @@ const SCENARIOS: SdkScenario[] = [
|
|
|
prompt: 'Prove that bash state persists. Then create {{cwd}}/note.txt with a tab-indented line, view it, replace that literal tab-indented line, and make the persistent shell exit with code 9.',
|
|
|
sessionId: 'persistent-tools-snapshot',
|
|
|
children: 0,
|
|
|
- configs: { live: persistentToolsLiveConfig, replay: persistentToolsReplayConfig },
|
|
|
+ configs: { live: minimalLiveConfig, replay: minimalReplayConfig },
|
|
|
expectedFiles: { 'note.txt': 'target:\n\tnew\n' },
|
|
|
expectedTools: { bash: ['command'], str_replace_editor: ['command', 'path'] },
|
|
|
+ expectedSystem: MINIMAL_SYSTEM_PROMPT,
|
|
|
+ expectedToolDescriptions: { bash: MINIMAL_BASH_DESCRIPTION },
|
|
|
policyContext: {
|
|
|
includes: ['Current DSH file policy: danger-full-access.', 'file modifications by available operations'],
|
|
|
excludes: ['write and edit tools', 'terminal sessions', 'one-shot bash commands'],
|
|
|
@@ -125,16 +141,33 @@ async function persistedLogs(sessionsRoot: string): Promise<PersistedLog[]> {
|
|
|
|
|
|
interface LoggedRequestHeader {
|
|
|
type?: string
|
|
|
- data?: { header?: { system?: unknown; tools?: Array<{ name: string; parameters: { required?: string[] } }> } }
|
|
|
+ data?: { header?: { system?: unknown; tools?: LoggedTool[] } }
|
|
|
}
|
|
|
|
|
|
-function assembledToolRequirements(log: PersistedLog): Record<string, string[]> {
|
|
|
+interface LoggedTool {
|
|
|
+ readonly name: string
|
|
|
+ readonly description?: unknown
|
|
|
+ readonly parameters: { readonly required?: string[] }
|
|
|
+}
|
|
|
+
|
|
|
+function assembledTools(log: PersistedLog): LoggedTool[] {
|
|
|
const event = log.content.trimEnd().split('\n')
|
|
|
.map(line => JSON.parse(line) as LoggedRequestHeader)
|
|
|
.find(candidate => candidate.type === 'request/header')
|
|
|
const tools = event?.data?.header?.tools
|
|
|
if (tools === undefined) throw new Error('session log has no request/header tools')
|
|
|
- return Object.fromEntries(tools.map(tool => [tool.name, tool.parameters.required ?? []]))
|
|
|
+ return tools
|
|
|
+}
|
|
|
+
|
|
|
+function assembledToolRequirements(log: PersistedLog): Record<string, string[]> {
|
|
|
+ return Object.fromEntries(assembledTools(log).map(tool => [tool.name, tool.parameters.required ?? []]))
|
|
|
+}
|
|
|
+
|
|
|
+function assembledToolDescriptions(log: PersistedLog): Record<string, string> {
|
|
|
+ return Object.fromEntries(assembledTools(log).map((tool) => {
|
|
|
+ if (typeof tool.description !== 'string') throw new Error(`tool ${tool.name} has no description`)
|
|
|
+ return [tool.name, tool.description]
|
|
|
+ }))
|
|
|
}
|
|
|
|
|
|
function assembledSystem(log: PersistedLog): string {
|
|
|
@@ -400,6 +433,16 @@ describe('TypeScript SDK snapshots over the jsonrpc runtime', () => {
|
|
|
if (parent === undefined) throw new Error(`${scenario.name} has no parent session log`)
|
|
|
expect(assembledToolRequirements(parent)).toEqual(scenario.expectedTools)
|
|
|
}
|
|
|
+ if (scenario.expectedSystem !== undefined) {
|
|
|
+ const parent = ordered[0]
|
|
|
+ if (parent === undefined) throw new Error(`${scenario.name} has no parent session log`)
|
|
|
+ expect(assembledSystem(parent)).toBe(scenario.expectedSystem)
|
|
|
+ }
|
|
|
+ if (scenario.expectedToolDescriptions !== undefined) {
|
|
|
+ const parent = ordered[0]
|
|
|
+ if (parent === undefined) throw new Error(`${scenario.name} has no parent session log`)
|
|
|
+ expect(assembledToolDescriptions(parent)).toMatchObject(scenario.expectedToolDescriptions)
|
|
|
+ }
|
|
|
if (scenario.policyContext !== undefined) {
|
|
|
const parent = ordered[0]
|
|
|
if (parent === undefined) throw new Error(`${scenario.name} has no parent session log`)
|