c-cpp.ts 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626
  1. import type { Node as SyntaxNode } from 'web-tree-sitter';
  2. import { getChildByField, getNodeText } from '../tree-sitter-helpers';
  3. import type { LanguageExtractor } from '../tree-sitter-types';
  4. /**
  5. * Find the function NAME's `qualified_identifier` (`Foo::bar`) inside a
  6. * declarator, skipping the `parameter_list` — a parameter with a qualified type
  7. * (`const std::string& x`) must NOT be mistaken for the method name. Without the
  8. * skip, a plain free function `std::string TableFileName(const std::string&...)`
  9. * was named `string` (from the parameter type), so calls to it never resolved
  10. * and its file looked like nothing depended on it.
  11. */
  12. function findDeclaratorQualifiedId(declarator: SyntaxNode): SyntaxNode | undefined {
  13. const queue: SyntaxNode[] = [declarator];
  14. while (queue.length > 0) {
  15. const current = queue.shift()!;
  16. if (current.type === 'qualified_identifier') return current;
  17. for (let i = 0; i < current.namedChildCount; i++) {
  18. const child = current.namedChild(i);
  19. // Don't descend into parameters or the trailing return type — their types
  20. // (`const std::string&`, `-> std::string`) aren't the function name.
  21. if (child && child.type !== 'parameter_list' && child.type !== 'trailing_return_type') {
  22. queue.push(child);
  23. }
  24. }
  25. }
  26. return undefined;
  27. }
  28. /**
  29. * Recover the real function name from the macro-definition idiom
  30. * `MACRO_NAME(real_name, typed args…) { body }` — flash-attention's
  31. * `DEFINE_FLASH_FORWARD_KERNEL(flash_fwd_kernel, bool Is_dropout, …) { … }`
  32. * being the motivating case: tree-sitter parses the invocation as a
  33. * function_definition NAMED after the macro, so every such kernel shared one
  34. * name (`DEFINE_FLASH_FORWARD_KERNEL`) and the launch sites' calls to the real
  35. * names (`flash_fwd_kernel<…><<<…>>>`) could never resolve.
  36. *
  37. * Deliberately narrow so name-in-first-arg is unambiguous — ALL of:
  38. * - the parsed name is macro-shaped: ALL-CAPS with at least one underscore
  39. * (`TEST` never matches; K&R C definitions have lowercase names);
  40. * - the first "parameter" is a LONE identifier (no type, no declarator)
  41. * containing a lowercase letter — the name being defined;
  42. * - at least one more parameter follows and NONE of them is another lone
  43. * identifier — a second bare arg means the first isn't the name (gtest's
  44. * `TEST_F(Fixture, Name)`, `PYBIND11_MODULE(ext, m)`,
  45. * google-benchmark's `BENCHMARK_DEFINE_F(Fix, name)` all bail here).
  46. */
  47. function recoverCppMacroDefinedName(node: SyntaxNode, source: string): string | undefined {
  48. if (node.type !== 'function_definition') return undefined;
  49. const declarator = getChildByField(node, 'declarator');
  50. if (declarator?.type !== 'function_declarator') return undefined;
  51. const inner = getChildByField(declarator, 'declarator');
  52. if (inner?.type !== 'identifier') return undefined;
  53. const macroName = getNodeText(inner, source);
  54. if (!/^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+$/.test(macroName)) return undefined;
  55. const params = getChildByField(declarator, 'parameters');
  56. if (!params || params.namedChildCount < 2) return undefined;
  57. const loneIdentText = (p: SyntaxNode): string | null =>
  58. p.type === 'parameter_declaration' &&
  59. p.namedChildCount === 1 &&
  60. p.namedChild(0)?.type === 'type_identifier'
  61. ? getNodeText(p.namedChild(0)!, source)
  62. : null;
  63. const first = params.namedChild(0);
  64. const name = first ? loneIdentText(first) : null;
  65. if (!name || !/[a-z]/.test(name)) return undefined;
  66. for (let i = 1; i < params.namedChildCount; i++) {
  67. const p = params.namedChild(i);
  68. if (p && loneIdentText(p) !== null) return undefined;
  69. }
  70. return name;
  71. }
  72. function extractCppQualifiedMethodName(node: SyntaxNode, source: string): string | undefined {
  73. const macroDefined = recoverCppMacroDefinedName(node, source);
  74. if (macroDefined) return macroDefined;
  75. const declarator = getChildByField(node, 'declarator');
  76. if (!declarator) return undefined;
  77. const qid = findDeclaratorQualifiedId(declarator);
  78. if (!qid) return undefined;
  79. const parts = getNodeText(qid, source).trim().split('::').filter(Boolean);
  80. return parts[parts.length - 1];
  81. }
  82. function extractCppReceiverType(node: SyntaxNode, source: string): string | undefined {
  83. const declarator = getChildByField(node, 'declarator');
  84. if (!declarator) return undefined;
  85. const qid = findDeclaratorQualifiedId(declarator);
  86. if (!qid) return undefined;
  87. const parts = getNodeText(qid, source).trim().split('::').filter(Boolean);
  88. return parts.length > 1 ? parts.slice(0, -1).join('::') : undefined;
  89. }
  90. /**
  91. * Built-in / non-class return types that can never be a method receiver. We
  92. * store no `returnType` for these so resolution never tries to resolve a method
  93. * on `void` / `int` / etc.
  94. */
  95. const CPP_NON_CLASS_RETURN = new Set([
  96. 'void', 'bool', 'char', 'short', 'int', 'long', 'float', 'double', 'unsigned',
  97. 'signed', 'size_t', 'ssize_t', 'auto', 'wchar_t', 'char8_t', 'char16_t',
  98. 'char32_t', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t',
  99. 'uint32_t', 'uint64_t', 'intptr_t', 'uintptr_t', 'nullptr_t',
  100. ]);
  101. /**
  102. * Normalize a C++ return type to the bare class name a method could be called
  103. * on. Unwraps smart-pointer / optional wrappers to their element type
  104. * (`std::unique_ptr<Widget>` → `Widget`) so a factory's `->method()` resolves on
  105. * the pointee. Strips cv-qualifiers, `&`/`*`, namespace qualifiers, and other
  106. * template args. Returns undefined for primitives / void / `auto` / empty.
  107. */
  108. export function normalizeCppReturnType(raw: string): string | undefined {
  109. let t = raw.trim();
  110. if (!t) return undefined;
  111. // Unwrap smart pointers / optional to their pointee (the thing you call `->` on).
  112. const wrapper = t.match(/\b(?:std\s*::\s*)?(?:unique_ptr|shared_ptr|weak_ptr|optional)\s*<\s*([^,>]+?)\s*>/);
  113. if (wrapper && wrapper[1]) t = wrapper[1];
  114. t = t
  115. .replace(/\b(?:const|volatile|typename|struct|class|enum)\b/g, ' ')
  116. .replace(/<[^>]*>/g, ' ')
  117. .replace(/[*&]+/g, ' ')
  118. .replace(/\s+/g, ' ')
  119. .trim();
  120. if (!t) return undefined;
  121. const last = t.split('::').filter(Boolean).pop();
  122. if (!last) return undefined;
  123. if (CPP_NON_CLASS_RETURN.has(last)) return undefined;
  124. if (!/^[A-Za-z_]\w*$/.test(last)) return undefined;
  125. return last;
  126. }
  127. /**
  128. * Strip C++ template arguments from a base-type reference name so it matches the
  129. * bare class/struct the template was DEFINED as. `template<typename T> class
  130. * Base { … }` is indexed as a node named `Base`, but a derived class
  131. * `class D : public Base<int>` records its base as the full `Base<int>` (and
  132. * `class Q : public ns::Tpl<int>` as `ns::Tpl<int>`) — neither name-matches
  133. * `Base` / `ns::Tpl`, so the `extends` edge never resolves and the derived class
  134. * looks like it inherits from nothing (#1043).
  135. *
  136. * Removes every balanced `<…>` group regardless of nesting or position, so
  137. * `Base<int>` → `Base`, `ns::Tpl<Foo<int>>` → `ns::Tpl`, and the rare
  138. * `Outer<int>::Inner` → `Outer::Inner`. The remaining qualified head is exactly
  139. * what the non-templated base case already produces, so resolution treats them
  140. * identically. A name with no template args passes through unchanged.
  141. */
  142. export function stripCppTemplateArgs(name: string): string {
  143. if (!name.includes('<')) return name;
  144. let out = '';
  145. let depth = 0;
  146. for (const ch of name) {
  147. if (ch === '<') depth++;
  148. else if (ch === '>') { if (depth > 0) depth--; }
  149. else if (depth === 0) out += ch;
  150. }
  151. return out.trim();
  152. }
  153. /**
  154. * A function/method's return type lives in the `function_definition`'s `type`
  155. * field (`Metrics& Metrics::instance()` → `Metrics`). Constructors, destructors,
  156. * and conversion operators have no `type` field → undefined.
  157. */
  158. function extractCppReturnType(node: SyntaxNode, source: string): string | undefined {
  159. const typeNode = getChildByField(node, 'type');
  160. if (!typeNode) return undefined;
  161. return normalizeCppReturnType(getNodeText(typeNode, source));
  162. }
  163. export const cExtractor: LanguageExtractor = {
  164. // CUDA in C-detected headers (content-gated blank; see preParseCSource).
  165. preParse: preParseCSource,
  166. // Universal net: recover a real name from any macro-mangled function name.
  167. recoverMangledName: recoverMangledCppName,
  168. functionTypes: ['function_definition'],
  169. classTypes: [],
  170. methodTypes: [],
  171. interfaceTypes: [],
  172. structTypes: ['struct_specifier'],
  173. enumTypes: ['enum_specifier'],
  174. enumMemberTypes: ['enumerator'],
  175. typeAliasTypes: ['type_definition'], // typedef
  176. importTypes: ['preproc_include'],
  177. callTypes: ['call_expression'],
  178. variableTypes: ['declaration'],
  179. nameField: 'declarator',
  180. bodyField: 'body',
  181. paramsField: 'parameters',
  182. // A `const`/`static const` file-scope declaration carries a `type_qualifier`
  183. // child reading "const" — extract those as `constant`, plain globals as
  184. // `variable`.
  185. isConst: (node) =>
  186. node.namedChildren.some(
  187. (c: SyntaxNode) => c.type === 'type_qualifier' && c.text === 'const'
  188. ),
  189. getReturnType: extractCppReturnType,
  190. resolveTypeAliasKind: (node, _source) => {
  191. // C typedef: `typedef enum { ... } name;` or `typedef struct { ... } name;`
  192. // The inner enum_specifier/struct_specifier is anonymous, but we want the typedef name
  193. // to become the enum/struct node name.
  194. for (let i = 0; i < node.namedChildCount; i++) {
  195. const child = node.namedChild(i);
  196. if (!child) continue;
  197. if (child.type === 'enum_specifier' && getChildByField(child, 'body')) return 'enum';
  198. if (child.type === 'struct_specifier' && getChildByField(child, 'body')) return 'struct';
  199. }
  200. return undefined;
  201. },
  202. extractImport: (node, source) => {
  203. const importText = source.substring(node.startIndex, node.endIndex).trim();
  204. // C includes: #include <stdio.h>, #include "myheader.h"
  205. const systemLib = node.namedChildren.find((c: SyntaxNode) => c.type === 'system_lib_string');
  206. if (systemLib) {
  207. return { moduleName: getNodeText(systemLib, source).replace(/^<|>$/g, ''), signature: importText };
  208. }
  209. const stringLiteral = node.namedChildren.find((c: SyntaxNode) => c.type === 'string_literal');
  210. if (stringLiteral) {
  211. const stringContent = stringLiteral.namedChildren.find((c: SyntaxNode) => c.type === 'string_content');
  212. if (stringContent) {
  213. return { moduleName: getNodeText(stringContent, source), signature: importText };
  214. }
  215. }
  216. return null;
  217. },
  218. };
  219. /**
  220. * Detect tree-sitter's misparse of a macro-annotated class/struct, e.g.
  221. * `class MACRO Name { … }` or `class MACRO Name : public Base { … }` (#946).
  222. * Not knowing `MACRO` is a macro, tree-sitter reads `class MACRO` as an
  223. * *elaborated type specifier* (a bodyless `class_specifier`/`struct_specifier`
  224. * whose "type name" is the macro) and the rest as a function: `Name` becomes the
  225. * declarator and the `{ … }` a function body — so the whole declaration surfaces
  226. * as a `function_definition` named after the class, with a line range spanning
  227. * the entire class body. (A base clause, when present, additionally lands in an
  228. * `ERROR` node, but it isn't required — the leading macro alone triggers this.)
  229. *
  230. * Two structural signals pin it down with no risk to genuine code:
  231. * - the `type` field is a *bodyless* class/struct specifier — an elaborated
  232. * type, not a real inline-defined return type like
  233. * `struct P { int x; } makeP() { … }` (which carries a field list); and
  234. * - the declarator is not a `function_declarator` — a real function definition
  235. * always has one, which also leaves the legal-but-rare `class Foo f() { … }`
  236. * (an elaborated return type on a genuine function) alone.
  237. *
  238. * The class body is mangled by the same misparse and is unrecoverable, so —
  239. * matching how macro-prefixed C prototypes are handled — we drop the spurious
  240. * node rather than mint a misleading whole-body `function` that pollutes
  241. * callers/impact and skews kind statistics.
  242. */
  243. function isMacroMisparsedTypeDecl(node: SyntaxNode): boolean {
  244. const typeNode = getChildByField(node, 'type');
  245. if (!typeNode) return false;
  246. if (typeNode.type !== 'class_specifier' && typeNode.type !== 'struct_specifier') return false;
  247. if (typeNode.namedChildren.some((c: SyntaxNode) => c.type === 'field_declaration_list')) return false;
  248. const declarator = getChildByField(node, 'declarator');
  249. if (declarator && declarator.type === 'function_declarator') return false;
  250. return true;
  251. }
  252. /**
  253. * Blank an export/visibility macro in a `class/struct EXPORT_MACRO Name …`
  254. * *definition* header before parsing. Not knowing the macro, tree-sitter reads
  255. * `class EXPORT_MACRO` as an elaborated type specifier and the rest as a
  256. * function, so the whole class — its name, base clause, and members — drops out
  257. * of the index (#946 catches the resulting phantom function but can't recover
  258. * the class), which silently breaks type-hierarchy / inheritance-impact queries
  259. * for effectively every Unreal-Engine (`*_API`), Qt/Boost (`*_EXPORT`), LLVM
  260. * (`*_ABI`), … class. Replacing the macro with equal-length spaces preserves
  261. * every byte offset (and thus line/column), so the declaration then parses as a
  262. * normal class_specifier and the existing extraction emits the node, members,
  263. * and `extends` edge. (#1061, follow-up to #946.)
  264. *
  265. * Matched tightly so it can't touch the same macro used as an ordinary value
  266. * elsewhere (`int x = SOME_API;`): the macro is the ALL-CAPS token sitting
  267. * *between* `class`/`struct` and the type name, and the trailing `[:{]`
  268. * definition-guard fires only when a base clause or body follows — the only
  269. * shape that misparses. That guard also leaves elaborated-type variable
  270. * declarations (`struct FOO var;`, `class FOO obj = …`) untouched, since those
  271. * end in `;` / `=` / `[`, never `:` / `{`. C++-only (wired into cppExtractor),
  272. * so C's heavier use of `struct TAG var;` never reaches it.
  273. */
  274. export function blankCppExportMacros(source: string): string {
  275. if (source.indexOf('class') === -1 && source.indexOf('struct') === -1) return source;
  276. return source.replace(
  277. /\b(class|struct)(\s+)([A-Z][A-Z0-9_]+)(?=\s+[A-Za-z_]\w*(?:\s+final)?\s*[:{])/g,
  278. (_m, kw, ws, macro) => kw + ws + ' '.repeat(macro.length)
  279. );
  280. }
  281. /**
  282. * Blank a known inline-specifier macro sitting in front of a function's return
  283. * type (`FORCEINLINE FString GetName(…)`), before parsing. Not knowing the
  284. * macro, tree-sitter can't reconcile `MACRO <return-type> <name>(` — an extra
  285. * type-like token before the name — and drops into error recovery: the macro
  286. * becomes the return type and, for a non-primitive return, the return type gets
  287. * glued onto the name (`GetName` → `"FString GetName"`), so the function can't
  288. * be found by name and its callers don't link. This is pervasive in Unreal
  289. * Engine (`FORCEINLINE <ret> <name>(…)`) and in vendored third-party libraries
  290. * that define their own inline macro (pugixml's `PUGI__FN`, Godot's
  291. * `_FORCE_INLINE_`, Boost's `BOOST_FORCEINLINE`, …). Replacing the macro with
  292. * equal-length spaces preserves every byte offset (so line/column stay exact)
  293. * and the declaration then parses as an ordinary function — recovering the real
  294. * name AND the return type — mirroring how `blankCppExportMacros` recovers
  295. * macro-annotated classes (#946/#1061).
  296. *
  297. * Matched tightly so it can't touch an ordinary identifier: only the exact,
  298. * curated inline-specifier tokens below (never an arbitrary all-caps token, so a
  299. * real return type like `HRESULT DoIt()` is untouched), and only in specifier
  300. * position — immediately followed by whitespace and the identifier that starts
  301. * the return type or name. That lookahead leaves value/expression uses
  302. * (`x = FORCEINLINE ? …`), string literals, and longer words
  303. * (`FORCEINLINE_SOMETHINGELSE`, word-boundary) alone. To cover a new codebase's
  304. * inline macro, add its exact token to the list.
  305. */
  306. const CPP_INLINE_MACROS = [
  307. // Unreal Engine
  308. 'FORCEINLINE_DEBUGGABLE', 'FORCENOINLINE', 'FORCEINLINE',
  309. // pugixml (ubiquitous vendored XML parser): `#define PUGI__FN inline` before
  310. // the return type, plus `PUGIXML_FUNCTION` (linkage macro) between the return
  311. // type and the name — the blank mechanism handles both positions.
  312. 'PUGI__FN_NO_INLINE', 'PUGI__FN', 'PUGIXML_FUNCTION',
  313. // Godot
  314. '_ALWAYS_INLINE_', '_FORCE_INLINE_',
  315. // Boost
  316. 'BOOST_FORCEINLINE', 'BOOST_NOINLINE',
  317. // Qt (per-method markers + inline)
  318. 'Q_INVOKABLE', 'Q_SCRIPTABLE', 'Q_ALWAYS_INLINE', 'Q_SLOT', 'Q_SIGNAL',
  319. // Folly / Abseil / LLVM / V8 / Eigen / rapidjson
  320. 'FOLLY_ALWAYS_INLINE', 'FOLLY_NOINLINE',
  321. 'ABSL_ATTRIBUTE_ALWAYS_INLINE', 'ABSL_ATTRIBUTE_NOINLINE',
  322. 'LLVM_ATTRIBUTE_ALWAYS_INLINE', 'LLVM_ATTRIBUTE_NOINLINE',
  323. 'V8_INLINE', 'V8_NOINLINE',
  324. 'EIGEN_STRONG_INLINE', 'EIGEN_ALWAYS_INLINE', 'EIGEN_DEVICE_FUNC',
  325. 'RAPIDJSON_FORCEINLINE',
  326. // Mozilla / SpiderMonkey
  327. 'MOZ_ALWAYS_INLINE', 'MOZ_NEVER_INLINE',
  328. // Protocol Buffers
  329. 'PROTOBUF_ALWAYS_INLINE', 'PROTOBUF_NOINLINE',
  330. // {fmt} / spdlog
  331. 'FMT_CONSTEXPR20', 'FMT_CONSTEXPR', 'FMT_INLINE',
  332. // Hedley + nlohmann/json (bundles Hedley)
  333. 'JSON_HEDLEY_ALWAYS_INLINE', 'JSON_HEDLEY_NEVER_INLINE',
  334. 'HEDLEY_ALWAYS_INLINE', 'HEDLEY_NEVER_INLINE',
  335. // GLM (graphics math — pervasive in games/rendering)
  336. 'GLM_FUNC_QUALIFIER', 'GLM_FUNC_DECL', 'GLM_CONSTEXPR', 'GLM_INLINE',
  337. // Bullet Physics / Skia / OpenCV / EASTL / Cocos2d-x / Chromium-WebKit
  338. 'SIMD_FORCE_INLINE',
  339. 'SK_ALWAYS_INLINE',
  340. 'CV_ALWAYS_INLINE', 'CV_INLINE',
  341. 'EA_FORCE_INLINE', 'EA_NOINLINE',
  342. 'CC_INLINE',
  343. 'NEVER_INLINE',
  344. // C libraries: GLib, SQLite (internal linkage)
  345. 'G_INLINE_FUNC', 'SQLITE_PRIVATE', 'SQLITE_API',
  346. // Windows calling conventions (linkage position — recover the return type; the
  347. // name is salvaged regardless). Only the unambiguous, non-word-like ones.
  348. 'STDMETHODCALLTYPE', 'WINAPIV', 'WINAPI', 'APIENTRY',
  349. // Common cross-ecosystem inline/attribute hints
  350. 'ALWAYS_INLINE', 'FORCE_INLINE', 'NOINLINE',
  351. ] as const;
  352. // One alternation, longest token first so a longer macro wins over a prefix.
  353. const CPP_INLINE_MACRO_RE = new RegExp(
  354. `\\b(${[...CPP_INLINE_MACROS].sort((a, b) => b.length - a.length).join('|')})\\b(?=\\s+[A-Za-z_])`,
  355. 'g'
  356. );
  357. export function blankCppInlineMacros(source: string): string {
  358. if (!CPP_INLINE_MACROS.some((m) => source.indexOf(m) !== -1)) return source;
  359. return source.replace(CPP_INLINE_MACRO_RE, (m) => ' '.repeat(m.length));
  360. }
  361. // Bare C/C++ type/qualifier tokens that must never be taken as a recovered
  362. // function name (guards `recoverMangledCppName` against the `Ret (name)` idiom,
  363. // where the token before the params is the return type, not the name).
  364. const CPP_PRIMITIVE_NAMES = new Set([
  365. 'bool', 'void', 'int', 'char', 'short', 'long', 'float', 'double', 'unsigned',
  366. 'signed', 'wchar_t', 'char8_t', 'char16_t', 'char32_t', 'char_t', 'size_t',
  367. 'auto', 'const', 'struct', 'class', 'enum', 'union', 'typename',
  368. ]);
  369. /**
  370. * Universal fallback (any macro, no list) for a C/C++ function name still mangled
  371. * because a macro we don't blank sat in front of the return type: `MACRO Ret
  372. * name(…)` / `Ret MACRO name(…)` misparse so the return type is glued onto the
  373. * name ("Ret name", "char_t* to_str(double v)"). Recover the real identifier —
  374. * the token immediately before the parameter list (or the last token). This runs
  375. * AFTER the curated pre-parse blank, so it only ever sees the residual tail that
  376. * blanking didn't already fix cleanly (which also recovers the return type).
  377. *
  378. * Safe by construction: only touches an ALREADY-mangled name — one with an
  379. * internal space that isn't a legit `operator …`/destructor — so a well-formed
  380. * name is returned unchanged. Guarded against the two ways it could mis-pick:
  381. * the `Ret (name)` parenthesized-name idiom (left as-is, ambiguous), and a token
  382. * that is a bare primitive/keyword rather than a real identifier.
  383. */
  384. export function recoverMangledCppName(name: string): string {
  385. if (!/\s/.test(name) || name.startsWith('operator') || name.startsWith('~')) return name;
  386. if (/^\S+\s+\([A-Za-z_]\w*\)/.test(name)) return name; // `Ret (name)` idiom — leave alone
  387. const beforeParams = name.includes('(') ? name.slice(0, name.indexOf('(')) : name;
  388. const tokens = beforeParams.trim().split(/\s+/);
  389. const candidate = tokens[tokens.length - 1];
  390. if (!candidate || !/^[A-Za-z_]\w*$/.test(candidate) || CPP_PRIMITIVE_NAMES.has(candidate)) return name;
  391. return candidate;
  392. }
  393. /**
  394. * Blank Metal Shading Language `[[attribute]]` annotations before parsing.
  395. * MSL (≈ C++14) puts attributes AFTER the declarator — `float4 position
  396. * [[position]];`, `constant Uniforms &u [[buffer(0)]]` — a position
  397. * tree-sitter-cpp can't reconcile: a struct field with a trailing attribute
  398. * misparses into a shape that emits a spurious `extends` reference from the
  399. * struct to the field's *type* (`VertexIn extends float3`), which becomes a
  400. * wrong inheritance edge whenever the repo defines that type itself (simd
  401. * typedefs in a shared ShaderTypes.h are common). Replacing the attribute with
  402. * equal-length spaces preserves every byte offset and lets fields and
  403. * parameters parse as ordinary declarations, mirroring the macro blanks above.
  404. *
  405. * Matched tightly to the attribute shape — `[[ident]]`, `[[ident(args)]]`, and
  406. * comma-separated lists (`[[buffer(0), raster_order_group(0)]]`) — so a
  407. * subscripted lambda call (`arr[[]{ … }()]`, the only other way `[[` appears in
  408. * C++-family source) can never match: after `[[` a lambda continues with `]`,
  409. * never an identifier followed by `]]`. Applied ONLY to `.metal` files — in
  410. * regular C++ the pre-declarator attribute position (`[[nodiscard]] int f()`)
  411. * is legal syntax the grammar parses natively, and blanking it would be pure
  412. * blast radius. (#1121)
  413. */
  414. const METAL_ATTRIBUTE_RE =
  415. /\[\[\s*[A-Za-z_]\w*(?:\s*\([^()\n]*\))?(?:\s*,\s*[A-Za-z_]\w*(?:\s*\([^()\n]*\))?)*\s*\]\]/g;
  416. export function blankMetalAttributes(source: string): string {
  417. if (source.indexOf('[[') === -1) return source;
  418. return source.replace(METAL_ATTRIBUTE_RE, (m) => ' '.repeat(m.length));
  419. }
  420. /**
  421. * Blank CUDA-specific constructs before parsing `.cu`/`.cuh` files (parsed with
  422. * the C++ grammar). Three shapes tree-sitter-cpp can't reconcile, each replaced
  423. * with equal-length whitespace so every byte offset survives (#387):
  424. *
  425. * 1. Execution-space / storage specifiers: in `__global__ void step(…)` or
  426. * `__shared__ float tile[256]` the specifier parses as the declaration's
  427. * TYPE and shunts the real return/value type into an ERROR node — mangling
  428. * signatures and, for `__shared__` arrays, the declared name itself. Blanked
  429. * unconditionally (no following-token lookahead) so extended lambdas
  430. * (`[=] __device__ (int i) { … }`) recover too. `__restrict__` is deliberately
  431. * absent: the grammar already parses it natively as a type_qualifier.
  432. * 2. `__launch_bounds__(…)` between specifier and declarator — same misparse.
  433. * The parenthesized form is blanked first; a bare leftover token is caught
  434. * by the specifier list.
  435. * 3. Kernel-launch configs `step<<<grid, block, smem, stream>>>(args)`: the
  436. * chevrons lex as shift operators around an empty-named template, so no
  437. * call_expression exists and the host→kernel call edge — the main reason to
  438. * index CUDA at all — is lost. Blanking the `<<<…>>>` span leaves
  439. * `step (args)`, a plain call the grammar
  440. * parses natively (templated launches `k<T, 256><<<…>>>(…)` included).
  441. *
  442. * The launch-config match is deliberately bounded — statement/brace characters
  443. * excluded, span capped, newlines preserved by the replacer — so a stray `<<<`
  444. * (a committed merge-conflict marker, a string literal) can never blank a run
  445. * of real code: an unmatched launch degrades to the status quo for that call
  446. * site (no call edge), never to corruption. Applied to `.cu`/`.cuh` files and —
  447. * because much real CUDA lives in extension-less headers (cutlass launches the
  448. * majority of its kernels from `.h`; flash-attention's launch templates are
  449. * `.h`; llm.c keeps device helpers in C-detected `.h`) — to any C/C++-family
  450. * file whose CONTENT carries a strong CUDA marker (`looksLikeCudaSource`).
  451. * Unlike Metal's `[[attribute]]` (legal C++ syntax elsewhere, hence Metal's
  452. * strict extension gate), no CUDA marker is valid C++ anywhere: `<<<` isn't
  453. * legal syntax and the dunder specifiers are implementation-reserved names no
  454. * real codebase defines — so a content-triggered blank on a non-CUDA file can
  455. * only ever whitespace tokens inside comments or strings, which parse the same.
  456. */
  457. const CUDA_LAUNCH_BOUNDS_RE = /\b__launch_bounds__\s*\([^()\n]*\)/g;
  458. const CUDA_SPECIFIER_RE =
  459. /\b__(?:global|device|host|constant|shared|managed|grid_constant|forceinline|noinline|launch_bounds)__\b/g;
  460. // `;` stays excluded (launch configs are expressions; a stray `<<<` spanning
  461. // real statements always crosses one) and the span is capped. Braces are
  462. // allowed through the regex — `k<<<dim3{1,1,1}, dim3{256,1,1}>>>(…)` is a real
  463. // launch shape — but the replacer only blanks a BALANCED match: a merge
  464. // conflict's `<<<<<<< … >>>>>>>` region that dodges every `;` still opens
  465. // braces it never closes, so it fails the balance check and stays untouched.
  466. const CUDA_LAUNCH_CONFIG_RE = /<<<[^;]{0,400}?>>>/g;
  467. export function blankCudaConstructs(source: string): string {
  468. let out = source;
  469. if (out.indexOf('__') !== -1) {
  470. out = out
  471. .replace(CUDA_LAUNCH_BOUNDS_RE, (m) => ' '.repeat(m.length))
  472. .replace(CUDA_SPECIFIER_RE, (m) => ' '.repeat(m.length));
  473. }
  474. if (out.indexOf('<<<') !== -1) {
  475. out = out.replace(CUDA_LAUNCH_CONFIG_RE, (m) => {
  476. let depth = 0;
  477. for (let i = 0; i < m.length; i++) {
  478. const ch = m.charCodeAt(i);
  479. if (ch === 0x7b /* { */) depth++;
  480. else if (ch === 0x7d /* } */ && --depth < 0) return m;
  481. }
  482. return depth === 0 ? m.replace(/[^\n]/g, ' ') : m;
  483. });
  484. }
  485. return out;
  486. }
  487. /** Strong content markers for CUDA source in files without a CUDA extension
  488. * (headers). The dunders are execution-space specifiers that only nvcc defines;
  489. * `cudaStream_t` is the runtime's stream handle, pervasive in launcher headers
  490. * that themselves declare no kernel. Deliberately excludes weak markers (`dim3`,
  491. * `<<<`) that could plausibly appear in non-CUDA text. */
  492. function looksLikeCudaSource(source: string): boolean {
  493. return (
  494. source.indexOf('__global__') !== -1 ||
  495. source.indexOf('__device__') !== -1 ||
  496. source.indexOf('__constant__') !== -1 ||
  497. source.indexOf('cudaStream_t') !== -1
  498. );
  499. }
  500. /** C/C++ source pre-processing before tree-sitter: recover both macro-annotated
  501. * class definitions and macro-prefixed function definitions — plus the non-C++
  502. * surface of the dialects parsed with the C++ grammar: `.metal` MSL attribute
  503. * annotations, and CUDA specifiers + launch syntax (by `.cu`/`.cuh` extension
  504. * or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving. */
  505. function preParseCppSource(source: string, filePath?: string): string {
  506. const blanked = blankCppInlineMacros(blankCppExportMacros(source));
  507. const lower = filePath ? filePath.toLowerCase() : '';
  508. if (lower.endsWith('.metal')) return blankMetalAttributes(blanked);
  509. if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
  510. return blankCudaConstructs(blanked);
  511. }
  512. return blanked;
  513. }
  514. /** C source pre-processing: C-detected headers in CUDA projects (llm.c keeps
  515. * `__device__` helpers and kernel prototypes in plain `.h`) get the same
  516. * content-gated CUDA blank as C++. */
  517. function preParseCSource(source: string): string {
  518. return looksLikeCudaSource(source) ? blankCudaConstructs(source) : source;
  519. }
  520. export const cppExtractor: LanguageExtractor = {
  521. // Recover macro-annotated class/struct definitions (`class MYMODULE_API Foo : Base`,
  522. // #1061/#946) and macro-prefixed functions (`FORCEINLINE FString Foo()`, #1093
  523. // follow-up) that tree-sitter otherwise misparses.
  524. preParse: preParseCppSource,
  525. // Universal net for any macro the curated blank list misses.
  526. recoverMangledName: recoverMangledCppName,
  527. functionTypes: ['function_definition'],
  528. classTypes: ['class_specifier'],
  529. // A bodiless `class_specifier` is a forward declaration (`class Foo;`) or an
  530. // elaborated type reference, not a definition. Skip it so dozens of forward
  531. // decls across headers don't mint phantom `class` nodes that crowd out — and
  532. // get picked as the blast-radius representative over — the single real
  533. // definition, exactly as bodiless struct/enum specifiers are already skipped. (#1093)
  534. skipBodilessClass: true,
  535. methodTypes: ['function_definition'],
  536. interfaceTypes: [],
  537. structTypes: ['struct_specifier'],
  538. enumTypes: ['enum_specifier'],
  539. enumMemberTypes: ['enumerator'],
  540. typeAliasTypes: ['type_definition', 'alias_declaration'], // typedef and using
  541. importTypes: ['preproc_include'],
  542. callTypes: ['call_expression'],
  543. variableTypes: ['declaration'],
  544. nameField: 'declarator',
  545. bodyField: 'body',
  546. paramsField: 'parameters',
  547. resolveName: extractCppQualifiedMethodName,
  548. getReceiverType: extractCppReceiverType,
  549. getReturnType: extractCppReturnType,
  550. getVisibility: (node) => {
  551. // Check for access specifier in parent
  552. const parent = node.parent;
  553. if (parent) {
  554. for (let i = 0; i < parent.childCount; i++) {
  555. const child = parent.child(i);
  556. if (child?.type === 'access_specifier') {
  557. const text = child.text;
  558. if (text.includes('public')) return 'public';
  559. if (text.includes('private')) return 'private';
  560. if (text.includes('protected')) return 'protected';
  561. }
  562. }
  563. }
  564. return undefined;
  565. },
  566. resolveTypeAliasKind: (node, _source) => {
  567. // C++ typedef: `typedef enum { ... } name;` or `typedef struct { ... } name;`
  568. for (let i = 0; i < node.namedChildCount; i++) {
  569. const child = node.namedChild(i);
  570. if (!child) continue;
  571. if (child.type === 'enum_specifier' && getChildByField(child, 'body')) return 'enum';
  572. if (child.type === 'struct_specifier' && getChildByField(child, 'body')) return 'struct';
  573. }
  574. return undefined;
  575. },
  576. isMisparsedFunction: (name, node) => {
  577. // C++ macros like NLOHMANN_JSON_NAMESPACE_BEGIN cause tree-sitter to misparse
  578. // namespace blocks as function_definitions (e.g. name = "namespace detail").
  579. // Also filter C++ keywords that tree-sitter occasionally misinterprets as
  580. // function/method names (e.g. switch statements inside macro-confused scopes).
  581. if (name.startsWith('namespace')) return true;
  582. const cppKeywords = ['switch', 'if', 'for', 'while', 'do', 'case', 'return'];
  583. if (cppKeywords.includes(name)) return true;
  584. // `class MACRO Name : public Base { … }` misparses to a function_definition
  585. // named after the class. `blankCppExportMacros` (preParse) recovers the
  586. // common ALL-CAPS export-macro shape; this drop is the fallback for any
  587. // residual misparse it doesn't blank — still no phantom function (#1061/#946).
  588. return isMacroMisparsedTypeDecl(node);
  589. },
  590. extractImport: (node, source) => {
  591. const importText = source.substring(node.startIndex, node.endIndex).trim();
  592. // C++ includes: #include <iostream>, #include "myheader.h"
  593. const systemLib = node.namedChildren.find((c: SyntaxNode) => c.type === 'system_lib_string');
  594. if (systemLib) {
  595. return { moduleName: getNodeText(systemLib, source).replace(/^<|>$/g, ''), signature: importText };
  596. }
  597. const stringLiteral = node.namedChildren.find((c: SyntaxNode) => c.type === 'string_literal');
  598. if (stringLiteral) {
  599. const stringContent = stringLiteral.namedChildren.find((c: SyntaxNode) => c.type === 'string_content');
  600. if (stringContent) {
  601. return { moduleName: getNodeText(stringContent, source), signature: importText };
  602. }
  603. }
  604. return null;
  605. },
  606. };