1
0

smoke-python-runtime.py 95 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275
  1. #!/usr/bin/env python3
  2. """Keyless full-turn and snapshot smoke for the Python SDK runtime."""
  3. from __future__ import annotations
  4. import argparse
  5. import difflib
  6. import importlib
  7. import importlib.metadata
  8. import json
  9. import os
  10. import queue
  11. import re
  12. import secrets
  13. import shutil
  14. import subprocess
  15. import sys
  16. import sysconfig
  17. import tempfile
  18. import threading
  19. import time
  20. from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
  21. from pathlib import Path
  22. from typing import TYPE_CHECKING, Callable
  23. if TYPE_CHECKING:
  24. from deepseek_harness import RunResult
  25. EXPECTED_TEXT = "runtime smoke ok"
  26. LIVE_API_SENTINEL = "PYTHON_SDK_LIVE_OK"
  27. CODE_PROMPT = "Use run_code to compute the packaged worker smoke value."
  28. CODE_WORKER_TEXT = "code worker smoke ok"
  29. WORKFLOW_PROMPT = "Use workflow to compute the packaged worker smoke value without agents."
  30. WORKFLOW_WORKER_TEXT = "workflow worker smoke ok"
  31. MINIMAL_PROMPT = "Exercise the packaged minimal agent's persistent shell and string-replacement editor."
  32. MINIMAL_TEXT = "minimal agent smoke ok"
  33. MINIMAL_EDITOR_PATH_PREFIX = "Editor path: "
  34. FS_SEARCH_PROMPT = "Exercise the packaged filesystem search tools."
  35. FS_SEARCH_TEXT = "filesystem search smoke ok"
  36. FS_SEARCH_MARKER = "PACKAGED_FS_SEARCH_OK"
  37. MCP_PROMPT = "Exercise the packaged MCP client with one external stdio server."
  38. MCP_TEXT = "MCP client smoke ok"
  39. PROFILE_PLUGIN_PROMPT = "Verify the Python-installed dsh profile plugin."
  40. PROFILE_PLUGIN_TEXT = "profile plugin smoke ok"
  41. PROFILE_PLUGIN_MARKER = "PYTHON_INSTALLED_DSH_PROFILE_PLUGIN"
  42. IS_WINDOWS = sys.platform == "win32"
  43. MINIMAL_SHELL_TOOL = "pwsh" if IS_WINDOWS else "bash"
  44. MINIMAL_SHELL_COMMAND = (
  45. "$global:dshSdkCounter = [int]$global:dshSdkCounter + 1; "
  46. 'Write-Output "COUNT=$global:dshSdkCounter CWD=$((Get-Location).Path)"; '
  47. "if ($global:dshSdkCounter -eq 1) { Set-Location $env:TEMP }"
  48. if IS_WINDOWS
  49. else (
  50. "counter=$(( ${counter:-0} + 1 )); export counter; "
  51. "printf 'COUNT=%s CWD=%s\\n' \"$counter\" \"$PWD\"; "
  52. "if [ \"$counter\" -eq 1 ]; then cd /tmp; fi"
  53. )
  54. )
  55. MINIMAL_SHELL_SECOND_CWD = str(Path(tempfile.gettempdir()).resolve()) if IS_WINDOWS else "/tmp"
  56. SPAWN_NODE_PROMPT = "Run node --version through the packaged shell tool."
  57. SPAWN_NODE_TEXT = "spawn node smoke ok"
  58. SPAWN_NODE_CALL_ID = "spawn-node-shell"
  59. # The POSIX command string starts with `node ` inside the shell tool's `bash -c`
  60. # argv, the exact form @yao-pkg/pkg's unpatched SEA bootstrap rewrites to the
  61. # executable itself while stamping PKG_EXECPATH into the child environment.
  62. SPAWN_NODE_COMMAND = (
  63. 'node --version; if ($env:PKG_EXECPATH) { "PKG_EXECPATH=$env:PKG_EXECPATH" } else { "PKG_EXECPATH=ABSENT" }'
  64. if IS_WINDOWS
  65. else 'node --version; echo "PKG_EXECPATH=${PKG_EXECPATH:-ABSENT}"'
  66. )
  67. LEGACY_CUSTOM_DISABLED_ROWS = (
  68. "agent-instructions",
  69. "goal",
  70. "goal-round-driver",
  71. "command-goal",
  72. "plan-mode",
  73. "skill",
  74. "skill-filesystem",
  75. "tool-fs",
  76. "tool-fs-search",
  77. "tool-goal",
  78. "tool-ralph",
  79. "tool-skill",
  80. "tool-str-replace-editor",
  81. "tool-subagent-control",
  82. "tool-subagent-list-agents",
  83. "tool-subagent-fork",
  84. "tool-todo",
  85. "tool-web",
  86. )
  87. SNAPSHOT_PROMPT = "Run the advanced packaged-runtime snapshot scenario."
  88. SNAPSHOT_SESSION_ID = "advanced-executable"
  89. SNAPSHOT_DIRECT_CHILD_PROMPT = "Reply with exactly DIRECT_CHILD_OK and nothing else."
  90. SNAPSHOT_WORKFLOW_CHILD_PROMPT = "Reply with exactly WORKFLOW_CHILD_OK and nothing else."
  91. SNAPSHOT_FINAL_TEXT = "ADVANCED_EXECUTABLE_OK"
  92. RESTART_FIRST_PROMPT = "Complete the first isolated Python SDK process turn."
  93. RESTART_FIRST_TEXT = "PROCESS_ONE_OK"
  94. RESTART_SECOND_PROMPT = "Complete the second isolated Python SDK process turn."
  95. RESTART_SECOND_TEXT = "PROCESS_TWO_OK"
  96. RESTART_FIRST_SESSION_ID = "process-one"
  97. RESTART_SECOND_SESSION_ID = "process-two"
  98. SNAPSHOT_PLUGIN_CODE = """\
  99. return (ctx) => {
  100. harness.registerTool(ctx, harness.defineTool({
  101. name: 'snapshot_double',
  102. description: 'Double a number for executable snapshot verification.',
  103. parameters: { value: { type: 'number', required: true } },
  104. output: {
  105. schema: { type: 'number' },
  106. render(_args, value) {
  107. return [{ type: 'text', text: String(value) }]
  108. }
  109. },
  110. async execute(args) {
  111. return args.value * 2
  112. }
  113. }))
  114. }
  115. """
  116. SNAPSHOT_WORKFLOW_SCRIPT = (
  117. "phase('Delegate')\n"
  118. f"const reply = await agent('{SNAPSHOT_WORKFLOW_CHILD_PROMPT}', {{ label: 'workflow-child' }})\n"
  119. "return { reply }"
  120. )
  121. ADVANCED_SNAPSHOT_DIRECTORY = (
  122. Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "advanced"
  123. )
  124. ADVANCED_SNAPSHOT_FILENAMES = (
  125. "result.json", "session.v3.jsonl", "session.1.v3.jsonl", "session.2.v3.jsonl",
  126. )
  127. MINIMAL_SNAPSHOT_DIRECTORY = (
  128. Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "minimal"
  129. )
  130. if IS_WINDOWS:
  131. MINIMAL_SNAPSHOT_DIRECTORY /= "win-x64"
  132. MINIMAL_SNAPSHOT_FILENAMES = ("model-visible.json",)
  133. RESTART_SNAPSHOT_DIRECTORY = (
  134. Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "restart"
  135. )
  136. RESTART_SNAPSHOT_FILENAMES = (
  137. "result.json", "requests.json", "session.1.v3.jsonl", "session.2.v3.jsonl",
  138. )
  139. MCP_SERVER_SCRIPT = """\
  140. import json
  141. import os
  142. import sys
  143. import time
  144. log_path = os.environ.get("MCP_SMOKE_LOG")
  145. def send(message):
  146. sys.stdout.write(json.dumps(message, separators=(",", ":")) + "\\n")
  147. sys.stdout.flush()
  148. for line in sys.stdin:
  149. request = json.loads(line)
  150. if log_path is not None:
  151. with open(log_path, "a", encoding="utf-8") as log:
  152. log.write(str(request.get("method")) + "\\n")
  153. request_id = request.get("id")
  154. if request_id is None:
  155. continue
  156. method = request.get("method")
  157. if method == "initialize":
  158. send({
  159. "jsonrpc": "2.0",
  160. "id": request_id,
  161. "result": {
  162. "protocolVersion": request["params"]["protocolVersion"],
  163. "capabilities": {"tools": {"listChanged": False}},
  164. "serverInfo": {"name": "python-wheel-fixture", "version": "1.0.0"},
  165. },
  166. })
  167. elif method == "tools/list":
  168. # Keep discovery pending long enough that an SDK runtime answering
  169. # initialize before discovery completes makes its first model request
  170. # without this tool and fails deterministically.
  171. time.sleep(0.25)
  172. send({
  173. "jsonrpc": "2.0",
  174. "id": request_id,
  175. "result": {
  176. "tools": [{
  177. "name": "add",
  178. "description": "Add two numbers.",
  179. "inputSchema": {
  180. "type": "object",
  181. "properties": {"a": {"type": "number"}, "b": {"type": "number"}},
  182. "required": ["a", "b"],
  183. "additionalProperties": False,
  184. },
  185. }],
  186. },
  187. })
  188. elif method == "tools/call":
  189. params = request["params"]
  190. if params.get("name") != "add" or params.get("arguments") != {"a": 19, "b": 23}:
  191. send({
  192. "jsonrpc": "2.0",
  193. "id": request_id,
  194. "error": {"code": -32602, "message": "unexpected tool call"},
  195. })
  196. continue
  197. send({
  198. "jsonrpc": "2.0",
  199. "id": request_id,
  200. "result": {"content": [{"type": "text", "text": "42"}]},
  201. })
  202. else:
  203. send({
  204. "jsonrpc": "2.0",
  205. "id": request_id,
  206. "error": {"code": -32601, "message": f"unsupported method: {method}"},
  207. })
  208. """
  209. def write_profile_patch(
  210. root: Path,
  211. name: str,
  212. sessions: Path,
  213. patches: list[dict[str, object]],
  214. ) -> Path:
  215. """Write one JSON-form dsh profile patch with deterministic persistence."""
  216. path = root / name
  217. path.write_text(json.dumps([
  218. {
  219. "id": "session-persistence-jsonl",
  220. "config": {"root": str(sessions), "compression": "none"},
  221. },
  222. {"id": "session-telemetry-otel", "disabled": True},
  223. *patches,
  224. ], indent=2))
  225. return path
  226. def write_advanced_profile_patch(root: Path, name: str, sessions: Path) -> Path:
  227. """Write the shared custom, snapshot, and restart profile patch."""
  228. return write_profile_patch(root, name, sessions, [
  229. {"id": "tools", "config": {"mode": "both"}},
  230. {
  231. "id": "system-prompt",
  232. "config": {
  233. "persona": "You are a coding agent powered by the {{model}} model. Your working directory is {{cwd}}.",
  234. },
  235. },
  236. {"id": "session-log-deepseek", "config": {"enabled": True}},
  237. *({"id": row_id, "disabled": True} for row_id in LEGACY_CUSTOM_DISABLED_ROWS),
  238. {"id": "tool-bash", "disabled": True},
  239. {"id": "tool-pwsh", "disabled": True},
  240. {
  241. "id": "tool-subagent",
  242. "config": {
  243. "provider": "spawn",
  244. "toolName": "subagent",
  245. "backgroundMode": "one-shot",
  246. },
  247. },
  248. {"insert": [
  249. {"id": "code-runtime", "name": "@deepseek-ai/dsh-code-runtime-worker-thread"},
  250. {"id": "cordis-host-runner", "name": "@deepseek-ai/dsh-cordis-host-runner"},
  251. {"id": "cordis-tool", "name": "@deepseek-ai/dsh-tool-cordis"},
  252. ]},
  253. ])
  254. def write_mcp_patch(root: Path, sessions: Path, server_script: Path) -> Path:
  255. """Write a profile patch that mounts the packaged MCP client."""
  256. return write_profile_patch(root, "mcp.patch.yml", sessions, [{
  257. "insert": [{
  258. "id": "mcp-fixture",
  259. "name": "@deepseek-ai/dsh-mcp-client",
  260. "config": {
  261. "serverName": "fixture",
  262. "transport": "stdio",
  263. "command": sys.executable,
  264. "args": [str(server_script)],
  265. "env": {"MCP_SMOKE_LOG": str(server_script.with_suffix(".log"))},
  266. "failOnStartupError": True,
  267. "reconnect": {"enabled": False},
  268. },
  269. }],
  270. }])
  271. class MockModelHandler(BaseHTTPRequestHandler):
  272. """Return deterministic text, worker, and orchestration completions."""
  273. requests: list[dict[str, object]] = []
  274. def do_POST(self) -> None:
  275. content_length = int(self.headers.get("content-length", "0"))
  276. body = json.loads(self.rfile.read(content_length))
  277. self.requests.append(body)
  278. self.send_response(200)
  279. self.send_header("content-type", "text/event-stream")
  280. self.end_headers()
  281. chunks = completion_chunks(body)
  282. for chunk in chunks:
  283. self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
  284. self.wfile.write(b"data: [DONE]\n\n")
  285. self.wfile.flush()
  286. def log_message(self, _format: str, *_args: object) -> None:
  287. return
  288. def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
  289. """Choose the next deterministic model response from request history."""
  290. messages = body.get("messages")
  291. if not isinstance(messages, list) or not messages:
  292. raise AssertionError(f"model request has no messages: {body}")
  293. latest = messages[-1]
  294. if not isinstance(latest, dict):
  295. raise AssertionError(f"model request has an invalid latest message: {body}")
  296. if latest.get("role") == "tool":
  297. call_id, tool_name = latest_tool_call(messages)
  298. tool_text = message_text(latest.get("content"))
  299. mcp = mcp_tool_followup(call_id, tool_name, tool_text)
  300. if mcp is not None:
  301. return mcp
  302. fs_search = fs_search_tool_followup(call_id, tool_name, tool_text)
  303. if fs_search is not None:
  304. return fs_search
  305. spawn_node = spawn_node_tool_followup(call_id, tool_name, tool_text)
  306. if spawn_node is not None:
  307. return spawn_node
  308. minimal = minimal_tool_followup(body, call_id, tool_name, tool_text)
  309. if minimal is not None:
  310. return minimal
  311. advanced = advanced_tool_followup(body, call_id, tool_name, tool_text)
  312. if advanced is not None:
  313. return advanced
  314. if "42" not in tool_text:
  315. raise AssertionError(f"{tool_name} worker returned no expected value: {latest}")
  316. if tool_name == "run_code":
  317. return text_chunks(CODE_WORKER_TEXT)
  318. if tool_name == "workflow":
  319. return text_chunks(WORKFLOW_WORKER_TEXT)
  320. raise AssertionError(f"unexpected tool follow-up: {tool_name}")
  321. user_prompts = [
  322. message_text(message.get("content"))
  323. for message in reversed(messages)
  324. if isinstance(message, dict) and message.get("role") == "user"
  325. ]
  326. minimal_prompt = next(
  327. (
  328. prompt
  329. for prompt in user_prompts
  330. if prompt.startswith(f"{MINIMAL_PROMPT}\n{MINIMAL_EDITOR_PATH_PREFIX}")
  331. ),
  332. None,
  333. )
  334. # The minimal composition's assembled system prompt, advertised tool schemas, and
  335. # model-visible messages are pinned by its snapshot, not asserted here.
  336. if minimal_prompt is not None:
  337. return tool_call_chunks(
  338. "minimal-bash-1",
  339. MINIMAL_SHELL_TOOL,
  340. {"command": MINIMAL_SHELL_COMMAND},
  341. )
  342. scenario_prompts = {
  343. SNAPSHOT_DIRECT_CHILD_PROMPT,
  344. SNAPSHOT_WORKFLOW_CHILD_PROMPT,
  345. SNAPSHOT_PROMPT,
  346. CODE_PROMPT,
  347. WORKFLOW_PROMPT,
  348. FS_SEARCH_PROMPT,
  349. SPAWN_NODE_PROMPT,
  350. MCP_PROMPT,
  351. RESTART_FIRST_PROMPT,
  352. RESTART_SECOND_PROMPT,
  353. PROFILE_PLUGIN_PROMPT,
  354. }
  355. prompt = next(
  356. (candidate for candidate in user_prompts if candidate in scenario_prompts),
  357. message_text(latest.get("content")),
  358. )
  359. if prompt == SNAPSHOT_DIRECT_CHILD_PROMPT:
  360. return text_chunks("DIRECT_CHILD_OK")
  361. if prompt == SNAPSHOT_WORKFLOW_CHILD_PROMPT:
  362. return text_chunks("WORKFLOW_CHILD_OK")
  363. if prompt == SNAPSHOT_PROMPT:
  364. assert_advertised_tool(body, "cordis_define")
  365. return tool_call_chunks(
  366. "advanced-define",
  367. "cordis_define",
  368. {
  369. "plugin": {"kind": "new", "idPrefix": "snap"},
  370. "name": "Snapshot Double",
  371. "purpose": "Expose a deterministic doubling tool for executable snapshot verification.",
  372. "code": {"host": SNAPSHOT_PLUGIN_CODE},
  373. },
  374. )
  375. if prompt == RESTART_FIRST_PROMPT:
  376. return text_chunks(RESTART_FIRST_TEXT)
  377. if prompt == RESTART_SECOND_PROMPT:
  378. if any(
  379. isinstance(message, dict)
  380. and RESTART_FIRST_TEXT in message_text(message.get("content"))
  381. for message in messages
  382. ):
  383. raise AssertionError("second isolated process inherited the first process history")
  384. return text_chunks(RESTART_SECOND_TEXT)
  385. if prompt == CODE_PROMPT:
  386. assert_advertised_tool(body, "run_code")
  387. return tool_call_chunks(
  388. "call-code-worker",
  389. "run_code",
  390. {"code": "return 6 * 7", "description": "Compute the smoke value"},
  391. )
  392. if prompt == WORKFLOW_PROMPT:
  393. assert_advertised_tool(body, "workflow")
  394. return tool_call_chunks(
  395. "call-workflow-worker",
  396. "workflow",
  397. {
  398. "script": "return 6 * 7",
  399. "meta": {
  400. "name": "pkg-worker-smoke",
  401. "description": "exercise the packaged workflow worker",
  402. },
  403. },
  404. )
  405. if prompt == FS_SEARCH_PROMPT:
  406. assert_advertised_tool(body, "grep")
  407. assert_advertised_tool(body, "glob")
  408. return tool_call_chunks(
  409. "fs-search-grep",
  410. "grep",
  411. {"pattern": FS_SEARCH_MARKER, "path": "."},
  412. )
  413. if prompt == SPAWN_NODE_PROMPT:
  414. assert_advertised_tool(body, MINIMAL_SHELL_TOOL)
  415. return tool_call_chunks(
  416. SPAWN_NODE_CALL_ID,
  417. MINIMAL_SHELL_TOOL,
  418. {"command": SPAWN_NODE_COMMAND, "description": "Report the reachable Node version"},
  419. )
  420. if prompt == MCP_PROMPT:
  421. assert_advertised_tool(body, "mcp__fixture__add")
  422. return tool_call_chunks(
  423. "mcp-add",
  424. "mcp__fixture__add",
  425. {"a": 19, "b": 23},
  426. )
  427. if prompt == PROFILE_PLUGIN_PROMPT:
  428. system_text = "\n".join(
  429. message_text(message.get("content"))
  430. for message in messages
  431. if isinstance(message, dict) and message.get("role") == "system"
  432. )
  433. if PROFILE_PLUGIN_MARKER not in system_text:
  434. raise AssertionError("external profile plugin contributed no model-visible marker")
  435. return text_chunks(PROFILE_PLUGIN_TEXT)
  436. return text_chunks(EXPECTED_TEXT)
  437. def mcp_tool_followup(
  438. call_id: str,
  439. tool_name: str,
  440. tool_text: str,
  441. ) -> list[dict[str, object]] | None:
  442. """Verify one tool call through the packaged MCP client."""
  443. if call_id != "mcp-add":
  444. return None
  445. if tool_name != "mcp__fixture__add" or "42" not in tool_text:
  446. raise AssertionError(f"packaged MCP call returned an unexpected result: {tool_name}: {tool_text}")
  447. return text_chunks(MCP_TEXT)
  448. def fs_search_tool_followup(
  449. call_id: str,
  450. tool_name: str,
  451. tool_text: str,
  452. ) -> list[dict[str, object]] | None:
  453. """Exercise both ripgrep-backed tools through the packaged executable."""
  454. if not call_id.startswith("fs-search-"):
  455. return None
  456. if call_id == "fs-search-grep" and tool_name == "grep":
  457. if "needle.txt" not in tool_text or FS_SEARCH_MARKER not in tool_text:
  458. raise AssertionError(f"packaged grep returned no marker: {tool_text}")
  459. return tool_call_chunks(
  460. "fs-search-glob",
  461. "glob",
  462. {"pattern": "**/*.txt"},
  463. )
  464. if call_id == "fs-search-glob" and tool_name == "glob":
  465. if "needle.txt" not in tool_text:
  466. raise AssertionError(f"packaged glob returned no fixture path: {tool_text}")
  467. return text_chunks(FS_SEARCH_TEXT)
  468. raise AssertionError(f"unexpected filesystem-search follow-up: {call_id} {tool_name}: {tool_text}")
  469. def host_node_version() -> str:
  470. """The machine's own `node --version` line, the required shell resolution target."""
  471. node = shutil.which("node")
  472. if node is None:
  473. raise AssertionError("the spawn-node scenario requires Node on PATH for comparison")
  474. return subprocess.run(
  475. [node, "--version"], capture_output=True, text=True, check=True,
  476. ).stdout.strip()
  477. def spawn_node_tool_followup(
  478. call_id: str,
  479. tool_name: str,
  480. tool_text: str,
  481. ) -> list[dict[str, object]] | None:
  482. """Verify the packaged shell reached the machine's Node with a clean environment."""
  483. if call_id != SPAWN_NODE_CALL_ID:
  484. return None
  485. if tool_name != MINIMAL_SHELL_TOOL:
  486. raise AssertionError(f"spawn-node follow-up used an unexpected tool: {tool_name}")
  487. expected = host_node_version()
  488. if expected not in tool_text:
  489. raise AssertionError(
  490. f"packaged shell did not reach the machine's node {expected}: {tool_text}"
  491. )
  492. if "PKG_EXECPATH=ABSENT" not in tool_text:
  493. raise AssertionError(f"PKG_EXECPATH reached the shell child environment: {tool_text}")
  494. return text_chunks(SPAWN_NODE_TEXT)
  495. def minimal_tool_followup(
  496. body: dict[str, object],
  497. call_id: str,
  498. tool_name: str,
  499. tool_text: str,
  500. ) -> list[dict[str, object]] | None:
  501. """Verify the checked-in minimal composition's PTY and editor."""
  502. if not call_id.startswith("minimal-"):
  503. return None
  504. if call_id == "minimal-bash-1" and tool_name == MINIMAL_SHELL_TOOL:
  505. if "COUNT=1" not in tool_text:
  506. raise AssertionError(f"first persistent shell call lost its output: {tool_text}")
  507. return tool_call_chunks(
  508. "minimal-bash-2",
  509. MINIMAL_SHELL_TOOL,
  510. {"command": MINIMAL_SHELL_COMMAND},
  511. )
  512. if call_id == "minimal-bash-2" and tool_name == MINIMAL_SHELL_TOOL:
  513. expected = f"COUNT=2 CWD={MINIMAL_SHELL_SECOND_CWD}"
  514. if expected.lower() not in tool_text.lower():
  515. raise AssertionError(f"persistent shell did not retain state: {tool_text}")
  516. messages = body.get("messages")
  517. if not isinstance(messages, list):
  518. raise AssertionError("persistent editor smoke request has no messages")
  519. editor_path = next(
  520. (
  521. text.split(MINIMAL_EDITOR_PATH_PREFIX, 1)[1].strip()
  522. for message in messages
  523. if isinstance(message, dict) and message.get("role") == "user"
  524. for text in [message_text(message.get("content"))]
  525. if MINIMAL_EDITOR_PATH_PREFIX in text
  526. ),
  527. None,
  528. )
  529. if editor_path is None:
  530. raise AssertionError("persistent editor smoke prompt has no editor path")
  531. return tool_call_chunks(
  532. "minimal-editor",
  533. "str_replace_editor",
  534. {
  535. "command": "create",
  536. "path": editor_path,
  537. "file_text": "created by packaged editor\n",
  538. },
  539. )
  540. if call_id == "minimal-editor" and tool_name == "str_replace_editor":
  541. if "New file created successfully" not in tool_text:
  542. raise AssertionError(f"packaged editor did not create its file: {tool_text}")
  543. return text_chunks(MINIMAL_TEXT)
  544. raise AssertionError(f"unexpected minimal-agent follow-up: {call_id} {tool_name}: {tool_text}")
  545. def advanced_tool_followup(
  546. body: dict[str, object],
  547. call_id: str,
  548. tool_name: str,
  549. tool_text: str,
  550. ) -> list[dict[str, object]] | None:
  551. """Advance the executable snapshot's deterministic parent tool chain."""
  552. if not call_id.startswith("advanced-"):
  553. return None
  554. if call_id == "advanced-define" and tool_name == "cordis_define":
  555. if "Defined snap-1/pkg-1 (Snapshot Double)" not in tool_text:
  556. raise AssertionError(f"cordis_define returned no dynamic Package ids: {tool_text}")
  557. if "snapshot_double" in advertised_tool_names(body):
  558. raise AssertionError("snapshot_double was advertised before cordis_run")
  559. assert_advertised_tool(body, "cordis_run")
  560. return tool_call_chunks(
  561. "advanced-run",
  562. "cordis_run",
  563. {"pluginId": "snap-1", "packageId": "pkg-1", "mode": "run"},
  564. )
  565. if call_id == "advanced-run" and tool_name == "cordis_run":
  566. if "snap-1/pkg-1 is running (run-1)" not in tool_text:
  567. raise AssertionError(f"cordis_run returned no running Package ids: {tool_text}")
  568. assert_advertised_tool(body, "run_code")
  569. assert_advertised_tool(body, "snapshot_double")
  570. return tool_call_chunks(
  571. "advanced-code",
  572. "run_code",
  573. {
  574. "code": "return await tools.snapshot_double({ value: 21 })",
  575. "description": "Run the temporary Plugin tool",
  576. },
  577. )
  578. if call_id == "advanced-code" and tool_name == "run_code":
  579. if "42" not in tool_text:
  580. raise AssertionError(f"run_code returned no dynamic-tool value: {tool_text}")
  581. assert_advertised_tool(body, "subagent")
  582. return tool_call_chunks(
  583. "advanced-direct-child",
  584. "subagent",
  585. {
  586. "description": "Check direct child",
  587. "prompt": SNAPSHOT_DIRECT_CHILD_PROMPT,
  588. },
  589. )
  590. if call_id == "advanced-direct-child" and tool_name == "subagent":
  591. if "DIRECT_CHILD_OK" not in tool_text:
  592. raise AssertionError(f"subagent returned no expected child value: {tool_text}")
  593. assert_advertised_tool(body, "workflow")
  594. return tool_call_chunks(
  595. "advanced-workflow",
  596. "workflow",
  597. {
  598. "script": SNAPSHOT_WORKFLOW_SCRIPT,
  599. "meta": {
  600. "name": "advanced-exe-snapshot",
  601. "description": "exercise one packaged workflow child",
  602. },
  603. },
  604. )
  605. if call_id == "advanced-workflow" and tool_name == "workflow":
  606. if "WORKFLOW_CHILD_OK" not in tool_text:
  607. raise AssertionError(f"workflow returned no expected child value: {tool_text}")
  608. assert_advertised_tool(body, "cordis_undefine")
  609. return tool_call_chunks(
  610. "advanced-undefine",
  611. "cordis_undefine",
  612. {"pluginId": "snap-1"},
  613. )
  614. if call_id == "advanced-undefine" and tool_name == "cordis_undefine":
  615. if "Removed dynamic Plugin snap-1 and all of its Packages." not in tool_text:
  616. raise AssertionError(f"cordis_undefine returned no removal result: {tool_text}")
  617. if "snapshot_double" in advertised_tool_names(body):
  618. raise AssertionError("snapshot_double remained advertised after cordis_undefine")
  619. return text_chunks(SNAPSHOT_FINAL_TEXT)
  620. raise AssertionError(f"unexpected advanced tool follow-up: {call_id} {tool_name}: {tool_text}")
  621. def text_chunks(text: str) -> list[dict[str, object]]:
  622. """Build a complete streaming text response."""
  623. return [
  624. {"choices": [{"delta": {"role": "assistant", "content": None, "reasoning_content": ""}}]},
  625. {"choices": [{"delta": {"content": text}}]},
  626. {
  627. "choices": [{"delta": {"content": ""}, "finish_reason": "stop"}],
  628. "usage": {"prompt_tokens": 3, "completion_tokens": 3},
  629. },
  630. ]
  631. def tool_call_chunks(call_id: str, name: str, arguments: dict[str, object]) -> list[dict[str, object]]:
  632. """Build a complete streaming function-call response."""
  633. return [
  634. {"choices": [{"delta": {"role": "assistant", "content": None, "reasoning_content": ""}}]},
  635. {
  636. "choices": [{
  637. "delta": {
  638. "tool_calls": [{
  639. "index": 0,
  640. "id": call_id,
  641. "type": "function",
  642. "function": {"name": name, "arguments": json.dumps(arguments)},
  643. }],
  644. },
  645. }],
  646. },
  647. {
  648. "choices": [{"delta": {"content": ""}, "finish_reason": "tool_calls"}],
  649. "usage": {"prompt_tokens": 3, "completion_tokens": 3},
  650. },
  651. ]
  652. def latest_tool_call(messages: list[object]) -> tuple[str, str]:
  653. """Find the assistant call id and name paired with the latest tool result."""
  654. for message in reversed(messages[:-1]):
  655. if not isinstance(message, dict):
  656. continue
  657. calls = message.get("tool_calls")
  658. if not isinstance(calls, list):
  659. continue
  660. for call in reversed(calls):
  661. if not isinstance(call, dict):
  662. continue
  663. function = call.get("function")
  664. call_id = call.get("id")
  665. if (
  666. isinstance(call_id, str)
  667. and isinstance(function, dict)
  668. and isinstance(function.get("name"), str)
  669. ):
  670. return call_id, function["name"]
  671. raise AssertionError(f"tool result has no preceding assistant tool call: {messages}")
  672. def message_text(content: object) -> str:
  673. """Read OpenAI text content in either string or block-list form."""
  674. if isinstance(content, str):
  675. return content
  676. if isinstance(content, list):
  677. return "".join(
  678. block.get("text", "")
  679. for block in content
  680. if isinstance(block, dict) and isinstance(block.get("text"), str)
  681. )
  682. return ""
  683. def advertised_tool_names(body: dict[str, object]) -> set[str]:
  684. """Return the model-facing tool names advertised on one request."""
  685. tools = body.get("tools")
  686. if not isinstance(tools, list):
  687. raise AssertionError(f"model request advertised no tools: {body}")
  688. names: set[str] = set()
  689. for tool in tools:
  690. if not isinstance(tool, dict):
  691. continue
  692. function = tool.get("function")
  693. if isinstance(function, dict) and isinstance(function.get("name"), str):
  694. names.add(function["name"])
  695. return names
  696. def assert_advertised_tool(body: dict[str, object], expected: str) -> None:
  697. """Require the packaged deployment to expose the requested tool."""
  698. names = advertised_tool_names(body)
  699. if expected not in names:
  700. raise AssertionError(f"model request did not advertise {expected}: {names}")
  701. class MockModel:
  702. def __enter__(self) -> "MockModel":
  703. MockModelHandler.requests.clear()
  704. self.server = ThreadingHTTPServer(("127.0.0.1", 0), MockModelHandler)
  705. self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
  706. self.thread.start()
  707. host, port = self.server.server_address
  708. self.url = f"http://{host}:{port}"
  709. return self
  710. def __exit__(self, _exc_type: object, _exc: object, _tb: object) -> None:
  711. self.server.shutdown()
  712. self.server.server_close()
  713. self.thread.join(timeout=5)
  714. def main() -> None:
  715. parser = argparse.ArgumentParser(description=__doc__)
  716. parser.add_argument(
  717. "--scenario",
  718. choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-spawn-node", "sdk-mcp", "sdk-snapshot", "sdk-restart", "sdk-profile-plugin", "sdk-live", "direct"),
  719. default="all",
  720. )
  721. parser.add_argument("--exe", type=Path)
  722. parser.add_argument(
  723. "--installed-wheel",
  724. action="store_true",
  725. help="require a clean virtual environment containing matching installed SDK and runtime wheels",
  726. )
  727. parser.add_argument("--update-snapshots", action="store_true")
  728. args = parser.parse_args()
  729. if args.installed_wheel and args.exe is not None:
  730. parser.error("--installed-wheel resolves the wheel's own runtime and cannot be combined with --exe")
  731. if args.scenario == "sdk-live" and not args.installed_wheel:
  732. parser.error("--scenario sdk-live requires --installed-wheel")
  733. if args.scenario == "sdk-profile-plugin" and not args.installed_wheel:
  734. parser.error("--scenario sdk-profile-plugin requires --installed-wheel")
  735. if args.installed_wheel:
  736. args.exe = assert_installed_wheel_environment()
  737. if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-fs-search", "sdk-spawn-node", "sdk-snapshot", "sdk-restart", "direct"} and args.exe is None:
  738. parser.error("--exe is required for custom, minimal, snapshot, and direct scenarios")
  739. if args.update_snapshots and args.scenario not in {"all", "sdk-minimal", "sdk-snapshot", "sdk-restart"}:
  740. parser.error("--update-snapshots requires --scenario sdk-minimal, sdk-snapshot, sdk-restart, or all")
  741. if args.exe is not None and not args.exe.is_file():
  742. parser.error(f"runtime executable does not exist: {args.exe}")
  743. if args.scenario == "sdk-live":
  744. smoke_sdk_live()
  745. print("smoke-python-runtime: sdk-live passed")
  746. return
  747. with MockModel() as model:
  748. if args.scenario in {"all", "sdk-default"}:
  749. smoke_sdk_default(model.url)
  750. if args.scenario in {"all", "sdk-custom"}:
  751. assert args.exe is not None
  752. smoke_sdk_custom(model.url, args.exe.resolve())
  753. if args.scenario in {"all", "sdk-minimal"}:
  754. assert args.exe is not None
  755. smoke_sdk_minimal(model.url, args.exe.resolve(), args.update_snapshots)
  756. if args.scenario in {"all", "sdk-fs-search"}:
  757. assert args.exe is not None
  758. smoke_sdk_fs_search(model.url, args.exe.resolve())
  759. if args.scenario in {"all", "sdk-spawn-node"}:
  760. assert args.exe is not None
  761. smoke_sdk_spawn_node(model.url, args.exe.resolve())
  762. if args.scenario in {"all", "sdk-mcp"}:
  763. smoke_sdk_mcp(model.url, None if args.exe is None else args.exe.resolve())
  764. if args.scenario in {"all", "sdk-snapshot"}:
  765. assert args.exe is not None
  766. smoke_sdk_snapshot(model.url, args.exe.resolve(), args.update_snapshots)
  767. if args.scenario in {"all", "sdk-restart"}:
  768. assert args.exe is not None
  769. smoke_sdk_restart_snapshot(model.url, args.exe.resolve(), args.update_snapshots)
  770. if args.installed_wheel and args.scenario in {"all", "sdk-profile-plugin"}:
  771. smoke_sdk_profile_plugin(model.url)
  772. if args.scenario in {"all", "direct"}:
  773. assert args.exe is not None
  774. smoke_direct(model.url, args.exe.resolve())
  775. if not MockModelHandler.requests:
  776. raise AssertionError("mock model endpoint received no requests")
  777. print(f"smoke-python-runtime: {args.scenario} passed")
  778. def assert_installed_wheel_environment() -> Path:
  779. """Prove that this process imports matching non-editable wheel installations."""
  780. if sys.prefix == sys.base_prefix:
  781. raise AssertionError("installed-wheel smoke must run inside a virtual environment")
  782. if os.environ.get("PYTHONPATH"):
  783. raise AssertionError("installed-wheel smoke requires PYTHONPATH to be unset")
  784. if os.environ.get("DSH_RUNTIME_MODE"):
  785. raise AssertionError("installed-wheel smoke requires DSH_RUNTIME_MODE to be unset")
  786. repo_root = Path(__file__).resolve().parent.parent
  787. cwd = Path.cwd().resolve()
  788. if cwd.is_relative_to(repo_root):
  789. raise AssertionError(f"installed-wheel smoke must run outside the repository, got {cwd}")
  790. sdk_version = importlib.metadata.version("deepseek-harness-sdk")
  791. runtime_version = importlib.metadata.version("deepseek-harness-runtime-bin")
  792. if sdk_version != runtime_version:
  793. raise AssertionError(
  794. f"installed SDK/runtime versions differ: {sdk_version} != {runtime_version}"
  795. )
  796. expected_runtime_requirement = f"deepseek-harness-runtime-bin=={sdk_version}"
  797. requirements = importlib.metadata.requires("deepseek-harness-sdk") or []
  798. if expected_runtime_requirement not in requirements:
  799. raise AssertionError(
  800. f"installed SDK does not require {expected_runtime_requirement}: {requirements}"
  801. )
  802. prefix = Path(sys.prefix).resolve()
  803. imported: dict[str, Path] = {}
  804. for name in ("deepseek_harness", "deepseek_harness_runtime"):
  805. module = importlib.import_module(name)
  806. module_file = getattr(module, "__file__", None)
  807. if not isinstance(module_file, str):
  808. raise AssertionError(f"installed module {name} has no filesystem location")
  809. path = Path(module_file).resolve()
  810. if not path.is_relative_to(prefix):
  811. raise AssertionError(f"installed module {name} came from outside the virtual environment: {path}")
  812. if path.is_relative_to(repo_root):
  813. raise AssertionError(f"installed module {name} came from the repository checkout: {path}")
  814. imported[name] = path
  815. runtime_module = sys.modules["deepseek_harness_runtime"]
  816. executable = runtime_module.bundled_runtime_path().resolve()
  817. runtime_package = imported["deepseek_harness_runtime"].parent
  818. if not executable.is_relative_to(runtime_package):
  819. raise AssertionError(f"bundled runtime came from outside the installed runtime wheel: {executable}")
  820. runtime_files = importlib.metadata.files("deepseek-harness-runtime-bin") or []
  821. if not any(Path(file).name == executable.name for file in runtime_files):
  822. raise AssertionError(f"runtime executable is absent from installed distribution records: {executable}")
  823. return executable
  824. def smoke_sdk_live() -> None:
  825. """Run a real-model, tool-using two-turn task through installed wheels."""
  826. from deepseek_harness import DeepSeekHarness
  827. api_key = os.environ.get("DEEPSEEK_API_KEY")
  828. base_url = os.environ.get("DEEPSEEK_BASE_URL")
  829. if not api_key:
  830. raise AssertionError("sdk-live requires DEEPSEEK_API_KEY")
  831. if not base_url:
  832. raise AssertionError("sdk-live requires an explicit DEEPSEEK_BASE_URL")
  833. with tempfile.TemporaryDirectory(prefix="dsh-sdk-live-") as temporary:
  834. root = Path(temporary).resolve()
  835. dsh_home = root / "home"
  836. sessions = dsh_home / "sessions"
  837. marker = root / "live-api-marker.txt"
  838. session_id = "installed-wheel-live-api"
  839. shell_tool = "pwsh" if IS_WINDOWS else "bash"
  840. create_prompt = (
  841. f"Use the {shell_tool} tool to create the file at the absolute path below with exact UTF-8 "
  842. f"content {LIVE_API_SENTINEL}, with no newline or byte-order mark. "
  843. f"Then reply with exactly {LIVE_API_SENTINEL}.\n{marker}"
  844. )
  845. with DeepSeekHarness(
  846. provider="deepseek-official",
  847. model="deepseek-v4-flash",
  848. cwd=str(root),
  849. dsh_home=str(dsh_home),
  850. env={
  851. "DSH_PERMISSION_MODE": "danger-full-access",
  852. "DSH_TELEMETRY_DISABLED": "1",
  853. },
  854. api_key=api_key,
  855. base_url=base_url,
  856. request_timeout_seconds=180,
  857. ) as harness:
  858. created = harness.run(create_prompt, session_id=session_id)
  859. assert_live_turn("create", created)
  860. if not marker.is_file():
  861. raise AssertionError(f"create turn did not create {marker}")
  862. if marker.read_bytes() != LIVE_API_SENTINEL.encode("utf-8"):
  863. raise AssertionError(f"create turn wrote unexpected bytes to {marker}")
  864. # The challenge is absent from the prior turn and the verification prompt.
  865. challenge = secrets.token_hex(32).encode("ascii")
  866. marker.write_bytes(challenge)
  867. with tempfile.TemporaryDirectory(prefix="receipt-", dir=root) as receipt_directory:
  868. receipt = Path(receipt_directory) / "receipt.txt"
  869. verify_prompt = (
  870. "The file created in the previous turn has changed externally. "
  871. "Use a tool to read that same file and copy its exact current content to the "
  872. "new receipt path below, without changing the source file. "
  873. "Preserve every byte; do not add a newline or byte-order mark. "
  874. f"Then reply with exactly {LIVE_API_SENTINEL}.\n{receipt}"
  875. )
  876. verified = harness.run(verify_prompt, session_id=session_id)
  877. assert_live_turn("verify", verified)
  878. if not receipt.is_file():
  879. raise AssertionError(f"verify turn did not create receipt {receipt}")
  880. if receipt.read_bytes() != challenge:
  881. raise AssertionError(f"verify turn wrote unexpected bytes to receipt {receipt}")
  882. if not marker.is_file() or marker.read_bytes() != challenge:
  883. raise AssertionError(f"verify turn changed source file {marker}")
  884. assert_zstd_session_log(sessions)
  885. def assert_live_turn(label: str, result: RunResult) -> None:
  886. """Require completed model tool use and the exact smoke answer for each live turn."""
  887. if result.finish_reason != "completed":
  888. event_types = [event.get("type") for event in result.events]
  889. turn_end_data = next(
  890. (event.get("data") for event in reversed(result.events) if event.get("type") == "turn/end"),
  891. None,
  892. )
  893. turn_end = safe_turn_end(turn_end_data)
  894. raise AssertionError(
  895. f"{label} turn ended with {result.finish_reason!r}; "
  896. f"final={result.final_response!r}; turn_end={turn_end!r}; events={event_types}"
  897. )
  898. if not any(event.get("type") == "tool/call" for event in result.events):
  899. raise AssertionError(
  900. f"{label} turn made no model-requested tool call; "
  901. f"final={result.final_response!r}"
  902. )
  903. if result.final_response.strip() != LIVE_API_SENTINEL:
  904. raise AssertionError(f"{label} turn returned {result.final_response!r}")
  905. def safe_turn_end(value: object) -> object:
  906. """Project a live-provider failure without retaining credential-bearing text."""
  907. if not isinstance(value, dict):
  908. return value
  909. reason = value.get("reason")
  910. if not isinstance(reason, dict):
  911. return {"turn": value.get("turn"), "reason": reason}
  912. error = reason.get("error")
  913. safe_error = None
  914. if isinstance(error, dict):
  915. safe_error = {
  916. key: error.get(key)
  917. for key in ("code", "status")
  918. if error.get(key) is not None
  919. }
  920. return {
  921. "turn": value.get("turn"),
  922. "reason": {
  923. "kind": reason.get("kind"),
  924. **({"error": safe_error} if safe_error is not None else {}),
  925. },
  926. }
  927. def smoke_sdk_default(base_url: str) -> None:
  928. from deepseek_harness import DeepSeekHarness
  929. with tempfile.TemporaryDirectory(prefix="dsh-sdk-default-") as temporary:
  930. root = Path(temporary).resolve()
  931. dsh_home = root / "home"
  932. sessions = dsh_home / "sessions"
  933. with DeepSeekHarness(
  934. provider="deepseek-official",
  935. model="smoke-model",
  936. cwd=str(root),
  937. dsh_home=str(dsh_home),
  938. env={
  939. "DSH_PERMISSION_MODE": "danger-full-access",
  940. "DSH_TELEMETRY_DISABLED": "1",
  941. },
  942. api_key="sk-keyless-smoke",
  943. base_url=base_url,
  944. request_timeout_seconds=60,
  945. ) as harness:
  946. result = harness.run("reply with the smoke text", session_id="default-smoke")
  947. assert result.final_response == EXPECTED_TEXT, (
  948. f"final={result.final_response!r} finish={result.finish_reason!r} "
  949. f"events={[event.get('type') for event in result.events]!r} "
  950. f"turn_end={safe_turn_end(next((event.get('data', event) for event in reversed(result.events) if event.get('type') == 'turn/end'), {}))!r}"
  951. )
  952. assert_zstd_session_log(sessions)
  953. def smoke_sdk_custom(base_url: str, executable: Path) -> None:
  954. from deepseek_harness import DeepSeekHarness
  955. with tempfile.TemporaryDirectory(prefix="dsh-sdk-custom-") as temporary:
  956. root = Path(temporary).resolve()
  957. dsh_home = root / "home"
  958. sessions = dsh_home / "sessions"
  959. patch = write_advanced_profile_patch(root, "custom.patch.yml", sessions)
  960. with DeepSeekHarness(
  961. provider="deepseek-official",
  962. model="smoke-model",
  963. cwd=str(root),
  964. dsh_bin=str(executable),
  965. dsh_home=str(dsh_home),
  966. patches=(str(patch),),
  967. env={
  968. "DSH_PERMISSION_MODE": "danger-full-access",
  969. "DSH_TELEMETRY_DISABLED": "1",
  970. },
  971. api_key="sk-keyless-smoke",
  972. base_url=base_url,
  973. request_timeout_seconds=60,
  974. ) as harness:
  975. text_result = harness.run("reply with the smoke text", session_id="custom-smoke")
  976. code_result = harness.run(CODE_PROMPT, session_id="custom-smoke")
  977. workflow_result = harness.run(WORKFLOW_PROMPT, session_id="custom-smoke")
  978. assert text_result.final_response == EXPECTED_TEXT, text_result.final_response
  979. assert code_result.final_response == CODE_WORKER_TEXT, code_result.final_response
  980. assert workflow_result.final_response == WORKFLOW_WORKER_TEXT, workflow_result.final_response
  981. assert_session_log(sessions, root, EXPECTED_TEXT, CODE_WORKER_TEXT, WORKFLOW_WORKER_TEXT)
  982. def smoke_sdk_minimal(base_url: str, executable: Path, update_snapshots: bool) -> None:
  983. """Exercise the shipped standalone minimal profile through the packaged executable."""
  984. from deepseek_harness import DeepSeekHarness
  985. # One mock model serves every scenario of a run, so the snapshot takes this turn's slice.
  986. first_request = len(MockModelHandler.requests)
  987. with tempfile.TemporaryDirectory(prefix="dsh-sdk-minimal-") as temporary:
  988. root = Path(temporary).resolve()
  989. editor_path = root / "created.txt"
  990. prompt = f"{MINIMAL_PROMPT}\n{MINIMAL_EDITOR_PATH_PREFIX}{editor_path}"
  991. dsh_home = root / "home"
  992. sessions = dsh_home / "sessions"
  993. with DeepSeekHarness(
  994. provider="deepseek-official",
  995. model="smoke-model",
  996. cwd=str(root),
  997. dsh_bin=str(executable),
  998. dsh_home=str(dsh_home),
  999. profile="sdk-minimal",
  1000. api_key="sk-keyless-smoke",
  1001. base_url=base_url,
  1002. request_timeout_seconds=60,
  1003. ) as harness:
  1004. result = harness.run(prompt, session_id="minimal-agent-smoke")
  1005. event_text = json.dumps(result.events)
  1006. if MINIMAL_TEXT not in event_text:
  1007. raise AssertionError(f"minimal agent run emitted no final response: {result.events}")
  1008. if editor_path.read_text() != "created by packaged editor\n":
  1009. raise AssertionError(f"packaged editor wrote unexpected content: {editor_path.read_text()!r}")
  1010. assert_session_log(sessions, root, MINIMAL_TEXT, "COUNT=1", "COUNT=2")
  1011. files = build_minimal_snapshot_files(MockModelHandler.requests[first_request:], root)
  1012. compare_snapshot_files(
  1013. files, update_snapshots, MINIMAL_SNAPSHOT_DIRECTORY, MINIMAL_SNAPSHOT_FILENAMES,
  1014. )
  1015. def smoke_sdk_fs_search(base_url: str, executable: Path) -> None:
  1016. """Exercise real grep and glob spawns through the packaged executable."""
  1017. from deepseek_harness import DeepSeekHarness
  1018. with tempfile.TemporaryDirectory(prefix="dsh-sdk-fs-search-") as temporary:
  1019. root = Path(temporary).resolve()
  1020. (root / "needle.txt").write_text(f"{FS_SEARCH_MARKER}\n")
  1021. dsh_home = root / "home"
  1022. sessions = dsh_home / "sessions"
  1023. patch = write_profile_patch(root, "fs-search.patch.yml", sessions, [
  1024. {"id": "skill-filesystem", "disabled": True},
  1025. {"id": "tool-fs-search", "config": {"sampleOverCapGlobResults": False}},
  1026. ])
  1027. with DeepSeekHarness(
  1028. provider="deepseek-official",
  1029. model="smoke-model",
  1030. cwd=str(root),
  1031. dsh_bin=str(executable),
  1032. dsh_home=str(dsh_home),
  1033. patches=(str(patch),),
  1034. env={
  1035. "DSH_PERMISSION_MODE": "danger-full-access",
  1036. "DSH_TELEMETRY_DISABLED": "1",
  1037. },
  1038. api_key="sk-keyless-smoke",
  1039. base_url=base_url,
  1040. request_timeout_seconds=60,
  1041. ) as harness:
  1042. result = harness.run(FS_SEARCH_PROMPT, session_id="fs-search-smoke")
  1043. assert result.final_response == FS_SEARCH_TEXT, result.final_response
  1044. assert_session_log(sessions, root, FS_SEARCH_TEXT, FS_SEARCH_MARKER, "needle.txt")
  1045. def smoke_sdk_spawn_node(base_url: str, executable: Path) -> None:
  1046. """A shell command starting with `node` must reach the machine's Node, not the executable."""
  1047. from deepseek_harness import DeepSeekHarness
  1048. with tempfile.TemporaryDirectory(prefix="dsh-sdk-spawn-node-") as temporary:
  1049. root = Path(temporary).resolve()
  1050. dsh_home = root / "home"
  1051. sessions = dsh_home / "sessions"
  1052. patch = write_profile_patch(root, "spawn-node.patch.yml", sessions, [])
  1053. with DeepSeekHarness(
  1054. provider="deepseek-official",
  1055. model="smoke-model",
  1056. cwd=str(root),
  1057. dsh_bin=str(executable),
  1058. dsh_home=str(dsh_home),
  1059. patches=(str(patch),),
  1060. env={
  1061. "DSH_PERMISSION_MODE": "danger-full-access",
  1062. "DSH_TELEMETRY_DISABLED": "1",
  1063. },
  1064. api_key="sk-keyless-smoke",
  1065. base_url=base_url,
  1066. request_timeout_seconds=60,
  1067. ) as harness:
  1068. result = harness.run(SPAWN_NODE_PROMPT, session_id="spawn-node-smoke")
  1069. assert result.final_response == SPAWN_NODE_TEXT, result.final_response
  1070. assert_session_log(sessions, root, SPAWN_NODE_TEXT, "PKG_EXECPATH=ABSENT")
  1071. def smoke_sdk_mcp(base_url: str, executable: Path | None) -> None:
  1072. """Discover and call an external stdio MCP tool through the packaged client."""
  1073. from deepseek_harness import DeepSeekHarness
  1074. with tempfile.TemporaryDirectory(prefix="dsh-sdk-mcp-") as temporary:
  1075. root = Path(temporary).resolve()
  1076. dsh_home = root / "home"
  1077. sessions = dsh_home / "sessions"
  1078. server_script = root / "mcp_server.py"
  1079. server_script.write_text(MCP_SERVER_SCRIPT)
  1080. patch = write_mcp_patch(root, sessions, server_script)
  1081. discovery_log = server_script.with_suffix(".log")
  1082. with DeepSeekHarness(
  1083. provider="deepseek-official",
  1084. model="smoke-model",
  1085. cwd=str(root),
  1086. dsh_bin=None if executable is None else str(executable),
  1087. dsh_home=str(dsh_home),
  1088. patches=(str(patch),),
  1089. env={
  1090. "DSH_PERMISSION_MODE": "danger-full-access",
  1091. "DSH_TELEMETRY_DISABLED": "1",
  1092. },
  1093. api_key="sk-keyless-smoke",
  1094. base_url=base_url,
  1095. request_timeout_seconds=60,
  1096. ) as harness:
  1097. result = harness.run(MCP_PROMPT, session_id="mcp-smoke")
  1098. assert result.final_response == MCP_TEXT, result.final_response
  1099. assert discovery_log.read_text().splitlines() == [
  1100. "initialize",
  1101. "notifications/initialized",
  1102. "tools/list",
  1103. "tools/call",
  1104. ]
  1105. assert_session_log(sessions, root, MCP_TEXT, "mcp__fixture__add", "42")
  1106. def smoke_sdk_profile_plugin(base_url: str) -> None:
  1107. """Install an external bundle through Python's dsh command and load it in the SDK."""
  1108. from deepseek_harness import DeepSeekHarness
  1109. with tempfile.TemporaryDirectory(prefix="dsh-sdk-profile-plugin-") as temporary:
  1110. root = Path(temporary).resolve()
  1111. dsh_home = root / "home"
  1112. plugin = root / "plugin"
  1113. plugin.mkdir()
  1114. (plugin / "package.json").write_text(json.dumps({
  1115. "name": "dsh-python-blackbox-plugin",
  1116. "version": "1.0.0",
  1117. "private": True,
  1118. "type": "module",
  1119. "exports": "./index.js",
  1120. "peerDependencies": {"@deepseek-ai/cordis": "*"},
  1121. "dsh": {"bundle": {"patch": "./cordis.patch.yml"}},
  1122. }, indent=2))
  1123. (plugin / "index.js").write_text(
  1124. "import { Context } from '@deepseek-ai/cordis'\n"
  1125. "export const name = 'python-sdk-blackbox-plugin'\n"
  1126. "export const inject = ['systemPrompt']\n"
  1127. "export function apply(ctx) {\n"
  1128. " if (!(ctx instanceof Context)) throw new Error('external plugin loaded a second Cordis instance')\n"
  1129. " ctx.effect(() => ctx.systemPrompt.section({\n"
  1130. " name: 'python-sdk:blackbox-plugin',\n"
  1131. " order: 10,\n"
  1132. f" text: '{PROFILE_PLUGIN_MARKER}',\n"
  1133. " }))\n"
  1134. "}\n"
  1135. )
  1136. (plugin / "cordis.patch.yml").write_text(json.dumps([{
  1137. "insert": [{"id": "python-sdk-blackbox-plugin", "name": "dsh-python-blackbox-plugin"}],
  1138. }], indent=2))
  1139. dsh = Path(sysconfig.get_path("scripts")) / ("dsh.exe" if IS_WINDOWS else "dsh")
  1140. environment = {**os.environ, "DSH_HOME": str(dsh_home)}
  1141. installed = subprocess.run(
  1142. [str(dsh), "plugin", "--profile", "sdk", "add", f"file:{plugin}"],
  1143. cwd=root,
  1144. env=environment,
  1145. text=True,
  1146. capture_output=True,
  1147. check=False,
  1148. )
  1149. if installed.returncode != 0:
  1150. raise AssertionError(
  1151. f"Python-installed dsh could not add the external profile plugin: "
  1152. f"returncode={installed.returncode} (0x{installed.returncode & 0xffffffff:08x}) "
  1153. f"stdout={installed.stdout!r} stderr={installed.stderr!r}"
  1154. )
  1155. manifest = json.loads((dsh_home / "profiles" / "sdk" / "package.json").read_text())
  1156. if "dsh-python-blackbox-plugin" not in manifest.get("dependencies", {}):
  1157. raise AssertionError(f"dsh plugin did not record the external dependency: {manifest}")
  1158. if "dsh-python-blackbox-plugin" not in manifest["dsh"]["profile"]["bundles"]:
  1159. raise AssertionError(f"dsh plugin did not activate the external bundle: {manifest}")
  1160. harness = DeepSeekHarness(
  1161. provider="deepseek-official",
  1162. model="smoke-model",
  1163. cwd=str(root),
  1164. dsh_home=str(dsh_home),
  1165. env={
  1166. "DSH_PERMISSION_MODE": "danger-full-access",
  1167. "DSH_TELEMETRY_DISABLED": "1",
  1168. },
  1169. api_key="sk-keyless-smoke",
  1170. base_url=base_url,
  1171. request_timeout_seconds=60,
  1172. )
  1173. try:
  1174. with harness:
  1175. result = harness.run(PROFILE_PLUGIN_PROMPT, session_id="profile-plugin-smoke")
  1176. except Exception as error:
  1177. raise AssertionError(
  1178. f"external profile plugin runtime failed: {harness.client._runtime_diagnostics()}"
  1179. ) from error
  1180. assert result.final_response == PROFILE_PLUGIN_TEXT, result.final_response
  1181. assert_zstd_session_log(dsh_home / "sessions")
  1182. def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool) -> None:
  1183. """Drive and compare the advanced SDK/executable behavioral snapshot."""
  1184. from deepseek_harness import DeepSeekHarness
  1185. with tempfile.TemporaryDirectory(prefix="dsh-sdk-snapshot-") as temporary:
  1186. root = Path(temporary).resolve()
  1187. dsh_home = root / "home"
  1188. sessions = dsh_home / "sessions"
  1189. patch = write_advanced_profile_patch(root, "snapshot.patch.yml", sessions)
  1190. feedback_patch = write_profile_patch(root, "feedback.patch.yml", sessions, [{"insert": [
  1191. {"id": "snapshot-message-feedback", "name": "@deepseek-ai/dsh-message-feedback",
  1192. "config": {"maxNoteBytes": 1024}},
  1193. {"id": "snapshot-feedback-producer", "name": (
  1194. Path(__file__).resolve().parent.parent / "snapshots/sdk/text-turn/feedback-producer.mjs"
  1195. ).as_uri()},
  1196. ]}])
  1197. with DeepSeekHarness(
  1198. provider="deepseek-official",
  1199. model="smoke-model",
  1200. cwd=str(root),
  1201. dsh_bin=str(executable),
  1202. dsh_home=str(dsh_home),
  1203. patches=(str(patch), str(feedback_patch)),
  1204. env={
  1205. "DSH_PERMISSION_MODE": "danger-full-access",
  1206. "DSH_TELEMETRY_DISABLED": "1",
  1207. },
  1208. api_key="sk-keyless-smoke",
  1209. base_url=base_url,
  1210. request_timeout_seconds=60,
  1211. ) as harness:
  1212. result = harness.run(SNAPSHOT_PROMPT, session_id=SNAPSHOT_SESSION_ID)
  1213. assert result.final_response == SNAPSHOT_FINAL_TEXT, result.final_response
  1214. feedback_types = [event.get("type") for event in result.events
  1215. if str(event.get("type")).startswith("feedback/")]
  1216. if feedback_types != ["feedback/record", "feedback/message-put", "feedback/message-put", "feedback/message-delete"]:
  1217. raise AssertionError(f"advanced snapshot did not exercise all feedback mutations: {feedback_types}")
  1218. methods = [notification.method for notification in result.notifications]
  1219. if methods.count("subagent.started") != 2 or methods.count("subagent.finished") != 2:
  1220. raise AssertionError(f"advanced snapshot emitted unexpected subagent lifecycle: {methods}")
  1221. ptc_events = [event for event in result.events
  1222. if event.get("type") in ("tool/ptc-dispatch-start", "tool/ptc-dispatch")]
  1223. if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"]:
  1224. raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch events: {ptc_events}")
  1225. for event in ptc_events:
  1226. data = event["data"]
  1227. identity = (data.get("rootCallId"), data.get("parentCallId"), data.get("subCallId"))
  1228. if identity != ("advanced-code", "advanced-code", "advanced-code:ptc:1"):
  1229. raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch identity: {identity}")
  1230. logs = read_session_logs(sessions)
  1231. child_ids = snapshot_child_ids(result)
  1232. expected_ids = {SNAPSHOT_SESSION_ID, *child_ids}
  1233. if set(logs) != expected_ids:
  1234. raise AssertionError(f"advanced snapshot expected parent plus two child logs: {sorted(logs)}")
  1235. if "DIRECT_CHILD_OK" not in render_jsonl(logs[child_ids[0]]):
  1236. raise AssertionError("first advanced child log has no direct-subagent result")
  1237. if "WORKFLOW_CHILD_OK" not in render_jsonl(logs[child_ids[1]]):
  1238. raise AssertionError("second advanced child log has no workflow-subagent result")
  1239. files = build_snapshot_files(result, logs, child_ids, root)
  1240. compare_snapshot_files(
  1241. files, update_snapshots, ADVANCED_SNAPSHOT_DIRECTORY, ADVANCED_SNAPSHOT_FILENAMES,
  1242. )
  1243. def smoke_sdk_restart_snapshot(base_url: str, executable: Path, update_snapshots: bool) -> None:
  1244. """Snapshot two isolated sessions across complete SDK runtime restarts."""
  1245. from deepseek_harness import DeepSeekHarness
  1246. with tempfile.TemporaryDirectory(prefix="dsh-sdk-restart-") as temporary:
  1247. root = Path(temporary).resolve()
  1248. dsh_home = root / "home"
  1249. sessions = dsh_home / "sessions"
  1250. patch = write_advanced_profile_patch(root, "restart.patch.yml", sessions)
  1251. first_request = len(MockModelHandler.requests)
  1252. def run(prompt: str, session_id: str) -> "RunResult":
  1253. with DeepSeekHarness(
  1254. provider="deepseek-official",
  1255. model="smoke-model",
  1256. cwd=str(root),
  1257. dsh_bin=str(executable),
  1258. dsh_home=str(dsh_home),
  1259. patches=(str(patch),),
  1260. env={
  1261. "DSH_PERMISSION_MODE": "danger-full-access",
  1262. "DSH_TELEMETRY_DISABLED": "1",
  1263. },
  1264. api_key="sk-keyless-smoke",
  1265. base_url=base_url,
  1266. request_timeout_seconds=60,
  1267. ) as harness:
  1268. return harness.run(prompt, session_id=session_id)
  1269. first = run(RESTART_FIRST_PROMPT, RESTART_FIRST_SESSION_ID)
  1270. second = run(RESTART_SECOND_PROMPT, RESTART_SECOND_SESSION_ID)
  1271. requests = MockModelHandler.requests[first_request:]
  1272. if len(requests) != 2:
  1273. raise AssertionError(f"restart snapshot expected two model requests: {requests}")
  1274. if first.final_response != RESTART_FIRST_TEXT or second.final_response != RESTART_SECOND_TEXT:
  1275. raise AssertionError(
  1276. f"restart snapshot responses differ: {first.final_response!r}, {second.final_response!r}"
  1277. )
  1278. logs = read_session_logs(sessions)
  1279. expected_ids = {RESTART_FIRST_SESSION_ID, RESTART_SECOND_SESSION_ID}
  1280. if set(logs) != expected_ids:
  1281. raise AssertionError(f"restart snapshot expected two durable sessions: {sorted(logs)}")
  1282. for session_id, expected in (
  1283. (RESTART_FIRST_SESSION_ID, RESTART_FIRST_TEXT),
  1284. (RESTART_SECOND_SESSION_ID, RESTART_SECOND_TEXT),
  1285. ):
  1286. records = logs[session_id]
  1287. if sum(record.get("type") == "turn/end" for record in records) != 1:
  1288. raise AssertionError(f"restart snapshot {session_id} has an unexpected turn count")
  1289. if expected not in render_jsonl(records):
  1290. raise AssertionError(f"restart snapshot durable log has no {expected}")
  1291. files = build_restart_snapshot_files(
  1292. first,
  1293. second,
  1294. requests,
  1295. logs,
  1296. root,
  1297. sessions,
  1298. )
  1299. compare_snapshot_files(
  1300. files, update_snapshots, RESTART_SNAPSHOT_DIRECTORY, RESTART_SNAPSHOT_FILENAMES,
  1301. )
  1302. def smoke_direct(base_url: str, executable: Path) -> None:
  1303. with tempfile.TemporaryDirectory(prefix="dsh-direct-") as temporary:
  1304. root = Path(temporary).resolve()
  1305. dsh_home = root / "home"
  1306. sessions = dsh_home / "sessions"
  1307. patch = write_profile_patch(root, "direct.patch.yml", sessions, [])
  1308. environment = {
  1309. **os.environ,
  1310. "DSH_HOME": str(dsh_home),
  1311. "DSH_PERMISSION_MODE": "danger-full-access",
  1312. "DSH_TELEMETRY_DISABLED": "1",
  1313. "DEEPSEEK_API_KEY": "sk-keyless-smoke",
  1314. "DEEPSEEK_BASE_URL": base_url,
  1315. }
  1316. peer = RuntimePeer(
  1317. [str(executable), "--profile", "sdk", "--patch", str(patch)],
  1318. root,
  1319. environment,
  1320. )
  1321. try:
  1322. peer.send({"jsonrpc": "2.0", "id": "initialize", "method": "initialize", "params": {"cwd": str(root), "provider": "deepseek-official", "model": "smoke-model"}})
  1323. peer.read_until(lambda message: message.get("id") == "initialize")
  1324. peer.send({
  1325. "jsonrpc": "2.0",
  1326. "id": "prompt",
  1327. "method": "session/prompt",
  1328. "params": {"sessionId": "direct-smoke", "contentBlocks": [{"type": "text", "text": "reply with the smoke text"}]},
  1329. })
  1330. messages = peer.read_until(lambda message: message.get("id") == "prompt")
  1331. if not any(is_idle_notification(message) for message in messages):
  1332. messages.extend(peer.read_until(is_idle_notification))
  1333. event_text = json.dumps(messages)
  1334. if EXPECTED_TEXT not in event_text:
  1335. raise AssertionError(f"direct runtime emitted no final response: {messages}")
  1336. peer.send({"jsonrpc": "2.0", "id": "shutdown", "method": "shutdown"})
  1337. peer.read_until(lambda message: message.get("id") == "shutdown")
  1338. finally:
  1339. peer.close()
  1340. assert_session_log(sessions, root, EXPECTED_TEXT)
  1341. def is_idle_notification(message: dict[str, object]) -> bool:
  1342. """Return whether a JSON-RPC notification marks a session idle."""
  1343. params = message.get("params")
  1344. return (
  1345. message.get("method") == "session.status"
  1346. and isinstance(params, dict)
  1347. and params.get("status") == "idle"
  1348. )
  1349. class RuntimePeer:
  1350. def __init__(self, argv: list[str], cwd: Path, environment: dict[str, str]) -> None:
  1351. self.process = subprocess.Popen(
  1352. argv,
  1353. cwd=cwd,
  1354. env=environment,
  1355. stdin=subprocess.PIPE,
  1356. stdout=subprocess.PIPE,
  1357. stderr=subprocess.PIPE,
  1358. text=True,
  1359. encoding="utf-8",
  1360. bufsize=1,
  1361. )
  1362. self.stdout: queue.Queue[str | None] = queue.Queue()
  1363. self.stderr: list[str] = []
  1364. threading.Thread(target=self._read_stdout, daemon=True).start()
  1365. threading.Thread(target=self._read_stderr, daemon=True).start()
  1366. def send(self, message: dict[str, object]) -> None:
  1367. if self.process.stdin is None:
  1368. raise RuntimeError("runtime stdin is unavailable")
  1369. self.process.stdin.write(json.dumps(message) + "\n")
  1370. self.process.stdin.flush()
  1371. def read_until(self, predicate: Callable[[dict[str, object]], bool]) -> list[dict[str, object]]:
  1372. deadline = time.monotonic() + 60
  1373. messages: list[dict[str, object]] = []
  1374. while time.monotonic() < deadline:
  1375. try:
  1376. line = self.stdout.get(timeout=min(0.25, deadline - time.monotonic()))
  1377. except queue.Empty:
  1378. continue
  1379. if line is None:
  1380. raise RuntimeError(f"runtime exited before expected message; stderr: {''.join(self.stderr)}")
  1381. try:
  1382. message = json.loads(line)
  1383. except json.JSONDecodeError:
  1384. continue
  1385. messages.append(message)
  1386. if predicate(message):
  1387. return messages
  1388. raise TimeoutError(f"runtime timed out; messages={messages}; stderr={''.join(self.stderr)}")
  1389. def close(self) -> None:
  1390. if self.process.stdin is not None and not self.process.stdin.closed:
  1391. self.process.stdin.close()
  1392. try:
  1393. self.process.wait(timeout=10)
  1394. except subprocess.TimeoutExpired:
  1395. self.process.kill()
  1396. self.process.wait()
  1397. if self.process.returncode not in {0, -15}:
  1398. raise RuntimeError(f"runtime exited {self.process.returncode}; stderr: {''.join(self.stderr)}")
  1399. def _read_stdout(self) -> None:
  1400. assert self.process.stdout is not None
  1401. for line in self.process.stdout:
  1402. self.stdout.put(line)
  1403. self.stdout.put(None)
  1404. def _read_stderr(self) -> None:
  1405. assert self.process.stderr is not None
  1406. self.stderr.extend(self.process.stderr)
  1407. PERSISTED_SESSION_FILENAME = re.compile(r"^session(?:\.v([1-9]\d*))?\.jsonl(\.zstd)?$")
  1408. SNAPSHOT_SESSION_FILENAME = re.compile(
  1409. r"^session(?:\.([1-9]\d*))?(?:\.v([1-9]\d*))?\.jsonl$",
  1410. )
  1411. def persisted_session_filename_version(path: Path, compressed: bool = False) -> int | None:
  1412. """Return one canonical persistence basename's generation for the selected encoding."""
  1413. match = PERSISTED_SESSION_FILENAME.fullmatch(path.name)
  1414. if match is None or (match.group(2) is not None) != compressed:
  1415. return None
  1416. return int(match.group(1) or 0)
  1417. def latest_persisted_session_paths(sessions: Path, compressed: bool = False) -> list[Path]:
  1418. """Select the numeric-highest immutable generation in each physical Session directory."""
  1419. pattern = "*.jsonl.zstd" if compressed else "*.jsonl"
  1420. selected: dict[Path, tuple[int, Path]] = {}
  1421. for path in sessions.rglob(pattern):
  1422. version = persisted_session_filename_version(path, compressed)
  1423. if version is None:
  1424. continue
  1425. previous = selected.get(path.parent)
  1426. if previous is None or version > previous[0]:
  1427. selected[path.parent] = (version, path)
  1428. return sorted((entry[1] for entry in selected.values()), key=lambda path: str(path))
  1429. def session_header_version(content: str, label: str) -> int:
  1430. """Read a non-negative physical Session generation from the first JSONL record."""
  1431. first = next((line for line in content.splitlines() if line), None)
  1432. if first is None:
  1433. raise AssertionError(f"{label}: Session log is empty")
  1434. header = json.loads(first)
  1435. version = header.get("version") if isinstance(header, dict) and header.get("type") == "session" else None
  1436. if not isinstance(version, int) or isinstance(version, bool) or version < 0:
  1437. raise AssertionError(f"{label}: Session header has no non-negative integer version")
  1438. return version
  1439. def assert_current_session_version(version: int, label: str) -> None:
  1440. """Require generated logs to use the source writer generation, independent of goldens."""
  1441. source = Path(__file__).resolve().parents[1] / "packages/core/session/src/types.ts"
  1442. declarations = re.findall(
  1443. r"^export const SESSION_FORMAT_VERSION = ([0-9]+)$",
  1444. source.read_text(encoding="utf-8"),
  1445. re.MULTILINE,
  1446. )
  1447. if len(declarations) != 1:
  1448. raise AssertionError(f"{source}: expected one literal SESSION_FORMAT_VERSION declaration")
  1449. current_version = int(declarations[0])
  1450. if version != current_version:
  1451. raise AssertionError(
  1452. f"{label}: expected current Session format v{current_version}, got v{version}",
  1453. )
  1454. def assert_persisted_session_version(path: Path, content: str) -> int:
  1455. """Require generated persistence filenames and headers to use the current generation."""
  1456. filename_version = persisted_session_filename_version(path)
  1457. if filename_version is None:
  1458. raise AssertionError(f"non-canonical Session persistence filename: {path.name}")
  1459. header_version = session_header_version(content, path.name)
  1460. if filename_version != header_version:
  1461. raise AssertionError(
  1462. f"{path.name}: filename declares Session format v{filename_version}, "
  1463. f"header declares v{header_version}",
  1464. )
  1465. assert_current_session_version(header_version, path.name)
  1466. return header_version
  1467. def snapshot_session_filename(index: int, version: int) -> str:
  1468. """Render parent/ordinal snapshot role plus an omitted-v0 generation."""
  1469. if index < 0 or version < 0:
  1470. raise ValueError("snapshot Session index and version must be non-negative")
  1471. ordinal = "" if index == 0 else f".{index}"
  1472. generation = "" if version == 0 else f".v{version}"
  1473. return f"session{ordinal}{generation}.jsonl"
  1474. def parse_snapshot_session_filename(name: str) -> tuple[int, int] | None:
  1475. """Parse one canonical parent/ordinal snapshot filename."""
  1476. match = SNAPSHOT_SESSION_FILENAME.fullmatch(name)
  1477. if match is None:
  1478. if name.startswith("session") and name.endswith(".jsonl"):
  1479. raise AssertionError(f"invalid snapshot Session filename: {name}")
  1480. return None
  1481. return int(match.group(1) or 0), int(match.group(2) or 0)
  1482. def selected_snapshot_session_files(directory: Path) -> dict[int, Path]:
  1483. """Select one highest-generation expected file per parent/ordinal role."""
  1484. selected: dict[int, tuple[int, Path]] = {}
  1485. for path in directory.iterdir():
  1486. if not path.is_file():
  1487. continue
  1488. parsed = parse_snapshot_session_filename(path.name)
  1489. if parsed is None:
  1490. continue
  1491. index, version = parsed
  1492. content = path.read_text(encoding="utf-8")
  1493. header_version = session_header_version(content, path.name)
  1494. if header_version != version:
  1495. raise AssertionError(
  1496. f"{path.name}: filename declares Session format v{version}, header declares v{header_version}",
  1497. )
  1498. previous = selected.get(index)
  1499. if previous is None or version > previous[0]:
  1500. selected[index] = (version, path)
  1501. return {index: value[1] for index, value in selected.items()}
  1502. def assert_session_log(sessions: Path, cwd: Path, *expected_texts: str) -> None:
  1503. logs = latest_persisted_session_paths(sessions)
  1504. if len(logs) != 1:
  1505. raise AssertionError(f"expected one JSONL session log under {sessions}, found {logs}")
  1506. content = logs[0].read_text()
  1507. assert_persisted_session_version(logs[0], content)
  1508. lines = content.splitlines()
  1509. header = json.loads(lines[0])
  1510. if header.get("cwd") != str(cwd):
  1511. raise AssertionError(f"session header cwd is not absolute/canonical: {header}")
  1512. rendered = "\n".join(lines)
  1513. for expected in expected_texts:
  1514. if expected not in rendered:
  1515. raise AssertionError(f"session log has no {expected!r} response: {logs[0]}")
  1516. def assert_zstd_session_log(sessions: Path) -> None:
  1517. logs = latest_persisted_session_paths(sessions, compressed=True)
  1518. if len(logs) != 1:
  1519. raise AssertionError(f"expected one Zstandard JSONL session log under {sessions}, found {logs}")
  1520. if not logs[0].read_bytes().startswith(bytes.fromhex("28b52ffd")):
  1521. raise AssertionError(f"session log has no Zstandard magic: {logs[0]}")
  1522. def read_session_logs(sessions: Path) -> dict[str, list[dict[str, object]]]:
  1523. """Parse every persisted JSONL session into a map keyed by header id."""
  1524. logs: dict[str, list[dict[str, object]]] = {}
  1525. for path in latest_persisted_session_paths(sessions):
  1526. content = path.read_text(encoding="utf-8")
  1527. assert_persisted_session_version(path, content)
  1528. records = [
  1529. json.loads(line)
  1530. for line in content.splitlines()
  1531. if line
  1532. ]
  1533. if not records or records[0].get("type") != "session":
  1534. raise AssertionError(f"session log has no header: {path}")
  1535. session_id = records[0].get("id")
  1536. if not isinstance(session_id, str):
  1537. raise AssertionError(f"session log header has no string id: {path}")
  1538. if session_id in logs:
  1539. raise AssertionError(f"duplicate persisted session id: {session_id}")
  1540. logs[session_id] = records
  1541. return logs
  1542. def snapshot_child_ids(result: "RunResult") -> list[str]:
  1543. """Return the two child session ids in their SDK notification order."""
  1544. child_ids: list[str] = []
  1545. for notification in result.notifications:
  1546. if notification.method != "subagent.started":
  1547. continue
  1548. payload = notification.payload
  1549. if payload.get("parentSessionId") != SNAPSHOT_SESSION_ID:
  1550. continue
  1551. child_id = payload.get("childSessionId")
  1552. if isinstance(child_id, str) and child_id not in child_ids:
  1553. child_ids.append(child_id)
  1554. if len(child_ids) != 2:
  1555. raise AssertionError(f"advanced snapshot expected two child session ids: {child_ids}")
  1556. return child_ids
  1557. def build_minimal_snapshot_files(
  1558. requests: list[dict[str, object]],
  1559. cwd: Path,
  1560. ) -> dict[str, str]:
  1561. """Render the minimal composition's model-visible surface as expected output.
  1562. Every assembled system prompt, advertised tool schema, and system or user message is
  1563. kept verbatim: they carry what the deployment actually shows the model, so a plugin
  1564. that contributes an unintended system section or user message cannot pass unnoticed.
  1565. Assistant and tool payloads keep only their call identity because their text differs
  1566. across the platforms this expected output must replay on. The shipped profile omits
  1567. dynamic runtime context, so every message it emits is compared.
  1568. """
  1569. snapshot = []
  1570. for body in requests:
  1571. messages = body.get("messages")
  1572. if not isinstance(messages, list):
  1573. raise AssertionError(f"minimal model request has no messages: {body}")
  1574. snapshot.append({
  1575. "tools": minimal_snapshot_text(body.get("tools"), cwd),
  1576. "messages": [
  1577. minimal_snapshot_message(message, cwd)
  1578. for message in messages
  1579. ],
  1580. })
  1581. return {"model-visible.json": json.dumps(snapshot, indent=2, ensure_ascii=False) + "\n"}
  1582. def minimal_snapshot_message(message: object, cwd: Path) -> dict[str, object]:
  1583. """Reduce one model-visible message to its stable, behavior-carrying parts."""
  1584. if not isinstance(message, dict):
  1585. raise AssertionError(f"minimal model request has an invalid message: {message}")
  1586. role = message.get("role")
  1587. if role in ("system", "user"):
  1588. return {"role": role, "text": minimal_snapshot_text(message_text(message.get("content")), cwd)}
  1589. if role == "assistant":
  1590. calls = message.get("tool_calls")
  1591. if not isinstance(calls, list):
  1592. raise AssertionError(f"minimal assistant message has no tool calls: {message}")
  1593. return {
  1594. "role": role,
  1595. "toolCalls": [
  1596. {"id": call.get("id"), "name": (call.get("function") or {}).get("name")}
  1597. for call in calls
  1598. if isinstance(call, dict)
  1599. ],
  1600. }
  1601. if role == "tool":
  1602. return {"role": role, "toolCallId": message.get("tool_call_id"), "text": "{{tool-result}}"}
  1603. raise AssertionError(f"minimal model request has an unexpected message role: {message}")
  1604. def minimal_snapshot_text(value: object, cwd: Path) -> object:
  1605. """Replace the scenario's temporary working directory everywhere it appears."""
  1606. if isinstance(value, str):
  1607. return value.replace(str(cwd), "{{cwd}}")
  1608. if isinstance(value, list):
  1609. return [minimal_snapshot_text(item, cwd) for item in value]
  1610. if isinstance(value, dict):
  1611. return {key: minimal_snapshot_text(item, cwd) for key, item in value.items()}
  1612. return value
  1613. def build_snapshot_files(
  1614. result: "RunResult",
  1615. logs: dict[str, list[dict[str, object]]],
  1616. child_ids: list[str],
  1617. cwd: Path,
  1618. ) -> dict[str, str]:
  1619. """Render the SDK result and three persisted logs into stable expected outputs."""
  1620. replacements = [(str(cwd), "{{cwd}}"), (SNAPSHOT_SESSION_ID, "{{parent}}")]
  1621. replacements.append((snapshot_workflow_run_id(result), "{{workflow-run}}"))
  1622. for index, child_id in enumerate(child_ids, start=1):
  1623. replacements.append((child_id, f"{{{{child-{index}}}}}"))
  1624. agent_id = snapshot_agent_id(result, child_id)
  1625. replacements.append((agent_id, f"{{{{agent-{index}}}}}"))
  1626. command_index = 0
  1627. for record in logs[SNAPSHOT_SESSION_ID]:
  1628. data = record.get("data")
  1629. if record.get("type") == "command/run" and isinstance(data, dict):
  1630. command_index += 1
  1631. replacements.append((data["commandId"], f"{{{{command:{command_index}}}}}"))
  1632. if record.get("type") == "command/done" and isinstance(data, dict):
  1633. anonymous = re.search(r"Anonymous user: ([0-9a-f-]{36})", str(data.get("text")))
  1634. if anonymous is not None:
  1635. replacements.append((anonymous.group(1), "{{anonymous-user}}"))
  1636. feedback_targets = dict.fromkeys(
  1637. record["data"]["item"]["messageId"]
  1638. for record in logs[SNAPSHOT_SESSION_ID]
  1639. if record.get("type") == "feedback/message-put"
  1640. )
  1641. for index, message_id in enumerate(feedback_targets, start=1):
  1642. replacements.append((message_id, f"{{{{message:{index}}}}}"))
  1643. feedback_versions = dict.fromkeys(
  1644. record["data"]["item"]["version"]
  1645. for record in logs[SNAPSHOT_SESSION_ID]
  1646. if record.get("type") == "feedback/message-put"
  1647. )
  1648. for index, version in enumerate(feedback_versions, start=1):
  1649. replacements.append((version, f"{{{{feedback-version:{index}}}}}"))
  1650. replacements.sort(key=lambda pair: len(pair[0]), reverse=True)
  1651. result_value = {
  1652. "session_id": result.session_id,
  1653. "final_response": result.final_response,
  1654. "events": result.events,
  1655. "notifications": [
  1656. {"method": notification.method, "payload": notification.payload}
  1657. for notification in result.notifications
  1658. ],
  1659. }
  1660. normalized_result = normalize_snapshot_value(result_value, replacements)
  1661. parent_records = project_session_snapshot([
  1662. normalize_snapshot_value(record, replacements) for record in logs[SNAPSHOT_SESSION_ID]
  1663. ])
  1664. files = {
  1665. "result.json": json.dumps(normalized_result, indent=2, ensure_ascii=False) + "\n",
  1666. snapshot_session_filename(
  1667. 0, session_header_version(render_jsonl(parent_records), "advanced parent"),
  1668. ): render_jsonl(parent_records),
  1669. }
  1670. for index, child_id in enumerate(child_ids, start=1):
  1671. child_records = project_session_snapshot([
  1672. normalize_snapshot_value(record, replacements) for record in logs[child_id]
  1673. ])
  1674. child_content = render_jsonl(child_records)
  1675. files[snapshot_session_filename(
  1676. index, session_header_version(child_content, f"advanced child {index}"),
  1677. )] = child_content
  1678. return files
  1679. def build_restart_snapshot_files(
  1680. first: "RunResult",
  1681. second: "RunResult",
  1682. requests: list[dict[str, object]],
  1683. logs: dict[str, list[dict[str, object]]],
  1684. cwd: Path,
  1685. sessions: Path,
  1686. ) -> dict[str, str]:
  1687. """Render two SDK processes, isolated model histories, and durable logs."""
  1688. replacements = [
  1689. (str(sessions), "{{sessions}}"),
  1690. (str(cwd), "{{cwd}}"),
  1691. (RESTART_FIRST_SESSION_ID, "{{session-1}}"),
  1692. (RESTART_SECOND_SESSION_ID, "{{session-2}}"),
  1693. ]
  1694. result_value = [
  1695. {
  1696. "session_id": result.session_id,
  1697. "final_response": result.final_response,
  1698. "finish_reason": result.finish_reason,
  1699. "eventTypes": [
  1700. event.get("type")
  1701. for event in result.events
  1702. ],
  1703. "notificationMethods": [
  1704. notification.method
  1705. for notification in result.notifications
  1706. ],
  1707. }
  1708. for result in (first, second)
  1709. ]
  1710. request_value = [
  1711. {
  1712. "model": request.get("model"),
  1713. "messages": restart_request_messages(request),
  1714. "toolNames": sorted(advertised_tool_names(request)),
  1715. }
  1716. for request in requests
  1717. ]
  1718. first_records = project_session_snapshot([
  1719. normalize_snapshot_value(record, replacements) for record in logs[RESTART_FIRST_SESSION_ID]
  1720. ])
  1721. second_records = project_session_snapshot([
  1722. normalize_snapshot_value(record, replacements) for record in logs[RESTART_SECOND_SESSION_ID]
  1723. ])
  1724. first_content = render_jsonl(first_records)
  1725. second_content = render_jsonl(second_records)
  1726. return {
  1727. "result.json": json.dumps(
  1728. normalize_snapshot_value(result_value, replacements), indent=2, ensure_ascii=False,
  1729. ) + "\n",
  1730. "requests.json": json.dumps(
  1731. normalize_snapshot_value(request_value, replacements), indent=2, ensure_ascii=False,
  1732. ) + "\n",
  1733. snapshot_session_filename(
  1734. 1, session_header_version(first_content, "restart Session 1"),
  1735. ): first_content,
  1736. snapshot_session_filename(
  1737. 2, session_header_version(second_content, "restart Session 2"),
  1738. ): second_content,
  1739. }
  1740. def restart_request_messages(request: dict[str, object]) -> list[object]:
  1741. """Project model history while tokenizing composition-owned system prose."""
  1742. messages = request.get("messages")
  1743. if not isinstance(messages, list):
  1744. raise AssertionError(f"restart snapshot request has no messages: {request}")
  1745. return [
  1746. {"role": "system", "content": "{{system}}"}
  1747. if isinstance(message, dict) and message.get("role") == "system"
  1748. else message
  1749. for message in messages
  1750. ]
  1751. def snapshot_workflow_run_id(result: "RunResult") -> str:
  1752. """Return the one workflow run id emitted by the advanced scenario."""
  1753. run_ids: set[str] = set()
  1754. for event in result.events:
  1755. event_type = event.get("type")
  1756. data = event.get("data")
  1757. if not isinstance(event_type, str) or not event_type.startswith("tool-workflow/"):
  1758. continue
  1759. if isinstance(data, dict) and isinstance(data.get("runId"), str):
  1760. run_ids.add(data["runId"])
  1761. if len(run_ids) != 1:
  1762. raise AssertionError(f"advanced snapshot expected one workflow run id: {sorted(run_ids)}")
  1763. return next(iter(run_ids))
  1764. def snapshot_agent_id(result: "RunResult", child_id: str) -> str:
  1765. """Find the successful subagent id paired with one child session."""
  1766. for notification in result.notifications:
  1767. if notification.method != "subagent.finished":
  1768. continue
  1769. payload = notification.payload
  1770. if payload.get("childSessionId") != child_id:
  1771. continue
  1772. if payload.get("provider") != "spawn" or payload.get("status") != "ok":
  1773. raise AssertionError(f"advanced child did not finish successfully: {payload}")
  1774. agent_id = payload.get("agentId")
  1775. if isinstance(agent_id, str):
  1776. return agent_id
  1777. raise AssertionError(f"advanced snapshot has no finished agent for child {child_id}")
  1778. def normalize_snapshot_value(
  1779. value: object,
  1780. replacements: list[tuple[str, str]],
  1781. ) -> object:
  1782. """Scrub volatile values and bulky request headers without losing behavior."""
  1783. if isinstance(value, str):
  1784. normalized = value
  1785. for actual, token in replacements:
  1786. normalized = normalized.replace(actual, token)
  1787. return normalized
  1788. if isinstance(value, list):
  1789. return [normalize_snapshot_value(item, replacements) for item in value]
  1790. if not isinstance(value, dict):
  1791. return value
  1792. normalized = {
  1793. key: normalize_snapshot_value(item, replacements)
  1794. for key, item in value.items()
  1795. }
  1796. if normalized.get("type") == "session" and "createdAt" in normalized:
  1797. normalized["createdAt"] = 0
  1798. if "seq" in normalized and "time" in normalized:
  1799. normalized["time"] = 0
  1800. if normalized.get("type") in ("assistant/message", "assistant/attempt"):
  1801. data = normalized.get("data")
  1802. stream = data.get("stream") if isinstance(data, dict) else None
  1803. if isinstance(stream, list):
  1804. for member in stream:
  1805. if not isinstance(member, dict):
  1806. continue
  1807. if isinstance(member.get("time"), (int, float)):
  1808. member["time"] = 0
  1809. if isinstance(member.get("time0"), (int, float)):
  1810. member["time0"] = 0
  1811. dt = member.get("dt")
  1812. if isinstance(dt, list):
  1813. member["dt"] = [0] * len(dt)
  1814. if isinstance(normalized.get("id"), str) and normalized.get("role") in ("assistant", "system", "user"):
  1815. if not normalized["id"].startswith("{{message:"):
  1816. normalized["id"] = "{{messageId}}"
  1817. if normalized.get("type") in ("feedback/message-put", "feedback/message-delete"):
  1818. data = normalized.get("data")
  1819. if isinstance(data, dict):
  1820. item = data.get("item") if normalized["type"] == "feedback/message-put" else data
  1821. if isinstance(item, dict):
  1822. if normalized["type"] == "feedback/message-put":
  1823. item["createdAt"] = 0
  1824. item["updatedAt"] = 0
  1825. scrub_snapshot_header(normalized)
  1826. scrub_snapshot_system_message(normalized)
  1827. return normalized
  1828. def scrub_snapshot_header(value: dict[object, object]) -> None:
  1829. """Tokenize full request-header tool schemas while retaining tool names."""
  1830. data = value.get("data")
  1831. if not isinstance(data, dict):
  1832. return
  1833. if value.get("type") == "request/header":
  1834. header = data.get("header")
  1835. if not isinstance(header, dict):
  1836. return
  1837. tools = header.get("tools")
  1838. if isinstance(tools, list):
  1839. header["tools"] = [
  1840. tool.get("name") if isinstance(tool, dict) else "{{tools}}"
  1841. for tool in tools
  1842. ]
  1843. def scrub_snapshot_system_message(value: dict[object, object]) -> None:
  1844. """Tokenize the rendered prompt text of a `system/message` surface node."""
  1845. if value.get("type") != "system/message":
  1846. return
  1847. data = value.get("data")
  1848. message = data.get("message") if isinstance(data, dict) else None
  1849. content = message.get("content") if isinstance(message, dict) else None
  1850. if not isinstance(content, list):
  1851. return
  1852. for block in content:
  1853. if isinstance(block, dict) and block.get("type") == "text":
  1854. block["text"] = "{{system}}"
  1855. def render_jsonl(records: list[object]) -> str:
  1856. """Render parsed JSON values as compact, newline-terminated JSONL."""
  1857. return "".join(
  1858. json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n"
  1859. for record in records
  1860. )
  1861. def project_session_snapshot(records: list[dict[str, object]]) -> list[dict[str, object]]:
  1862. """Omit storage sequence/time envelopes from snapshot body records."""
  1863. projected = [dict(record) for record in records]
  1864. for record in projected[1:]:
  1865. for key in ("seq", "time", "seq0", "time0"):
  1866. record.pop(key, None)
  1867. return projected
  1868. SESSION_FORMAT_PROVENANCE = "{{sessionFormatVersion}}"
  1869. def expand_snapshot_stream_member(member: object) -> list[dict[str, object]]:
  1870. """Expand one compact Assistant stream member into logical provider chunks."""
  1871. if not isinstance(member, dict):
  1872. raise AssertionError(f"snapshot Assistant stream member is not an object: {member!r}")
  1873. member_type = member.get("type")
  1874. if member_type == "chunk":
  1875. chunk = member.get("chunk")
  1876. if not isinstance(chunk, dict):
  1877. raise AssertionError(f"snapshot Assistant chunk member has no chunk: {member!r}")
  1878. return [chunk]
  1879. packed_kinds = {
  1880. "text-chunks": ("texts", "text-delta", "text"),
  1881. "reasoning-chunks": ("texts", "reasoning-delta", "text"),
  1882. "tool-call-chunks": ("args", "tool-call-delta", "argumentsDelta"),
  1883. }
  1884. packed = packed_kinds.get(member_type)
  1885. if packed is None:
  1886. raise AssertionError(f"snapshot Assistant stream has unknown member type: {member_type!r}")
  1887. values_key, chunk_type, value_key = packed
  1888. values = member.get(values_key)
  1889. if not isinstance(values, list):
  1890. raise AssertionError(f"snapshot Assistant stream member has no {values_key}: {member!r}")
  1891. shared = {
  1892. key: member[key]
  1893. for key in ("index", "id", "name")
  1894. if key in member
  1895. }
  1896. return [
  1897. {"type": chunk_type, **shared, value_key: value}
  1898. for value in values
  1899. ]
  1900. def expand_snapshot_assistant_event(value: object) -> list[object]:
  1901. """Expand one direct or SDK-wrapped v2 settlement for generation-neutral comparison."""
  1902. if not isinstance(value, dict):
  1903. return [value]
  1904. event = value
  1905. wrapper_key: str | None = None
  1906. wrapper: dict[str, object] | None = None
  1907. if value.get("method") == "session.event":
  1908. for candidate in ("payload", "params"):
  1909. container = value.get(candidate)
  1910. nested = container.get("event") if isinstance(container, dict) else None
  1911. if isinstance(nested, dict):
  1912. event = nested
  1913. wrapper_key = candidate
  1914. wrapper = container
  1915. break
  1916. if event.get("type") not in ("assistant/message", "assistant/attempt"):
  1917. return [value]
  1918. data = event.get("data")
  1919. stream = data.get("stream") if isinstance(data, dict) else None
  1920. if not isinstance(stream, list):
  1921. return [value]
  1922. def wrap(expanded: dict[str, object]) -> object:
  1923. if wrapper_key is None or wrapper is None:
  1924. return expanded
  1925. return {**value, wrapper_key: {**wrapper, "event": expanded}}
  1926. common = {
  1927. key: data[key]
  1928. for key in ("turn", "step")
  1929. if key in data
  1930. }
  1931. expanded = [
  1932. wrap({
  1933. "type": "assistant/chunk",
  1934. "data": {**common, "chunk": chunk},
  1935. })
  1936. for member in stream
  1937. for chunk in expand_snapshot_stream_member(member)
  1938. ]
  1939. if event.get("type") == "assistant/message":
  1940. expanded.append(wrap({
  1941. **event,
  1942. "data": {key: item for key, item in data.items() if key != "stream"},
  1943. }))
  1944. return expanded
  1945. def normalize_session_format_comparison(
  1946. value: object,
  1947. source_session_version: int | None = None,
  1948. ) -> object:
  1949. """Canonicalize only generation provenance that differs across immutable Session files."""
  1950. if isinstance(value, list):
  1951. return [
  1952. normalize_session_format_comparison(expanded, source_session_version)
  1953. for item in value
  1954. for expanded in expand_snapshot_assistant_event(item)
  1955. ]
  1956. if not isinstance(value, dict):
  1957. return value
  1958. normalized = {
  1959. key: normalize_session_format_comparison(item, source_session_version)
  1960. for key, item in value.items()
  1961. }
  1962. if normalized.get("type") == "session" and "version" in normalized:
  1963. normalized["version"] = SESSION_FORMAT_PROVENANCE
  1964. normalized.setdefault("isSeeded", False)
  1965. ordered_header = {
  1966. key: normalized[key]
  1967. for key in ("type", "version", "id", "createdAt", "cwd", "isSeeded", "delegationDepth")
  1968. if key in normalized
  1969. }
  1970. normalized = {
  1971. **ordered_header,
  1972. **{key: item for key, item in normalized.items() if key not in ordered_header},
  1973. }
  1974. if isinstance(normalized.get("type"), str) and "data" in normalized:
  1975. normalized.pop("seq", None)
  1976. normalized.pop("time", None)
  1977. if source_session_version == 1 and normalized.get("type") == "assistant/message":
  1978. normalized.pop("sourceEventSeqs", None)
  1979. return normalized
  1980. def normalize_snapshot_comparison_text(name: str, content: str) -> str:
  1981. """Normalize Session generation provenance only while comparing committed expected outputs."""
  1982. if name.startswith("session") and name.endswith(".jsonl"):
  1983. parsed = [json.loads(line) for line in content.splitlines() if line]
  1984. header = parsed[0] if parsed else None
  1985. source_version = header.get("version") if isinstance(header, dict) else None
  1986. if not isinstance(source_version, int):
  1987. raise AssertionError(f"{name}: snapshot Session header has no integer format version")
  1988. records = [
  1989. normalize_session_format_comparison(expanded, source_version)
  1990. for record in parsed
  1991. for expanded in expand_snapshot_assistant_event(record)
  1992. ]
  1993. return render_jsonl(records)
  1994. if name.endswith(".json"):
  1995. return json.dumps(
  1996. normalize_session_format_comparison(json.loads(content)),
  1997. indent=2,
  1998. ensure_ascii=False,
  1999. ) + "\n"
  2000. return content
  2001. def compare_snapshot_files(
  2002. files: dict[str, str],
  2003. update: bool,
  2004. directory: Path,
  2005. filenames: tuple[str, ...],
  2006. ) -> None:
  2007. """Compare ordered artifact roles and Session content across generations, or write generated filenames."""
  2008. scenario = directory.name
  2009. def role_name(name: str) -> str:
  2010. parsed = parse_snapshot_session_filename(name)
  2011. return name if parsed is None else snapshot_session_filename(parsed[0], 0)
  2012. if tuple(map(role_name, files)) != tuple(map(role_name, filenames)):
  2013. raise AssertionError(f"{scenario} snapshot builder produced {tuple(files)}, expected {filenames}")
  2014. for name, content in files.items():
  2015. if parse_snapshot_session_filename(name) is not None:
  2016. assert_current_session_version(session_header_version(content, name), name)
  2017. if update:
  2018. directory.mkdir(parents=True, exist_ok=True)
  2019. for name, content in files.items():
  2020. (directory / name).write_text(content, encoding="utf-8", newline="\n")
  2021. print(f"smoke-python-runtime: updated snapshots in {directory}")
  2022. existing = [path for path in directory.iterdir() if path.is_file()] if directory.is_dir() else []
  2023. expected_non_session = {
  2024. name for name in filenames if parse_snapshot_session_filename(name) is None
  2025. }
  2026. existing_non_session = {
  2027. path.name for path in existing if parse_snapshot_session_filename(path.name) is None
  2028. }
  2029. if existing_non_session != expected_non_session:
  2030. raise AssertionError(
  2031. f"{scenario} snapshot files differ: "
  2032. f"missing={sorted(expected_non_session - existing_non_session)}, "
  2033. f"unexpected={sorted(existing_non_session - expected_non_session)}"
  2034. )
  2035. selected_expected = selected_snapshot_session_files(directory)
  2036. actual_sessions: dict[int, tuple[str, str]] = {}
  2037. for name, content in files.items():
  2038. parsed = parse_snapshot_session_filename(name)
  2039. if parsed is None:
  2040. continue
  2041. index, filename_version = parsed
  2042. header_version = session_header_version(content, name)
  2043. if filename_version != header_version:
  2044. raise AssertionError(
  2045. f"{name}: filename declares Session format v{filename_version}, "
  2046. f"header declares v{header_version}",
  2047. )
  2048. if index in actual_sessions:
  2049. raise AssertionError(f"{scenario} snapshot builder produced duplicate Session role {index}")
  2050. actual_sessions[index] = (name, content)
  2051. if set(selected_expected) != set(actual_sessions):
  2052. raise AssertionError(
  2053. f"{scenario} snapshot Session roles differ: "
  2054. f"expected={sorted(selected_expected)}, actual={sorted(actual_sessions)}",
  2055. )
  2056. for name, actual in files.items():
  2057. parsed = parse_snapshot_session_filename(name)
  2058. expected_path = directory / name if parsed is None else selected_expected[parsed[0]]
  2059. expected_text = expected_path.read_text(encoding="utf-8")
  2060. compared_actual = normalize_snapshot_comparison_text(name, actual)
  2061. compared_expected = normalize_snapshot_comparison_text(expected_path.name, expected_text)
  2062. if compared_actual == compared_expected:
  2063. continue
  2064. diff = "".join(difflib.unified_diff(
  2065. compared_expected.splitlines(keepends=True),
  2066. compared_actual.splitlines(keepends=True),
  2067. fromfile=f"expected/{expected_path.name}",
  2068. tofile=f"actual/{name}",
  2069. ))
  2070. raise AssertionError(
  2071. f"{scenario} executable snapshot mismatch in {name}; "
  2072. "rerun with --update-snapshots after reviewing the behavior\n"
  2073. f"{diff}"
  2074. )
  2075. if __name__ == "__main__":
  2076. main()