fix(extraction): skip raw strings in C++ macro blankers (#1505) (#1804)

Co-authored-by: Colby McHenry <colbymchenry@users.noreply.github.com>

CHANGELOG.md

@@ -139,6 +139,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 #### MCP / indexing
 
+- C++ functions following anonymous namespaces containing raw-string templates are now indexed correctly, even when template text resembles an unfinished macro call. (#1505)
+
 - Indexing now warns when parser errors leave a file with no symbols, including C++ raw strings with 16-character delimiters, so missing code is no longer silent. (#1522)
 
 - `codegraph index <path>` now refuses uninitialized paths and names the nearest initialized parent instead of silently rebuilding it; thanks @danusha2345. (#1524, #1689)

__tests__/cpp-raw-string-preparse-1505.test.ts

@@ -0,0 +1,177 @@
+import { beforeAll, describe, expect, it } from 'vitest';
+import { extractFromSource } from '../src/extraction';
+import { getParser, initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
+import {
+  blankCppAnnotationMacroCalls,
+  blankCppInlineAnnotationMacros,
+  blankCStatementMacroCalls,
+  blankCTypeKeywordArgs,
+  blankCFileScopePrefixedDeclMacros,
+  blankCParameterizedAnnotationMacros,
+  blankCDesignatedMacroArgs,
+  cExtractor,
+  cppExtractor,
+} from '../src/extraction/languages/c-cpp';
+
+// The original parses cleanly; #1505 is preParse corrupting its short delimiter,
+// distinct from the vendored grammar's 16-character delimiter error (#1522).
+function scaffoldSource(delimiter = 'GEN'): string {
+  return `#include <string>
+
+namespace {
+const char* kTpl = R"${delimiter}(
+DECLARE_THING(
+struct Ignored { int v; };
+int nested_fn() { return 1; }
+)${delimiter}";
+}
+
+int create_scaffold(int x) {
+  return x;
+}
+
+int helper_after(int y) {
+  return y + 1;
+}
+`;
+}
+
+const annotationBlankers = [
+  { name: 'line-leading annotations', blank: blankCppAnnotationMacroCalls },
+  { name: 'inline annotations', blank: blankCppInlineAnnotationMacros },
+];
+
+describe('C/C++ raw strings survive preParse (#1505)', () => {
+  beforeAll(async () => {
+    await initGrammars();
+    await loadGrammarsForLanguages(['cpp']);
+  });
+
+  it('indexes both functions after the anonymous namespace without introducing a parse error', () => {
+    const source = scaffoldSource();
+    const rewritten = cppExtractor.preParse!(source, 'scaffold.cpp');
+    for (const text of [source, rewritten]) {
+      const tree = getParser('cpp')!.parse(text)!;
+      try {
+        expect(tree.rootNode.hasError).toBe(false);
+      } finally {
+        tree.delete();
+      }
+    }
+    const result = extractFromSource('scaffold.cpp', source);
+    expect(result.nodes.filter((node) => node.kind === 'function').map((node) => node.name))
+      .toEqual(['create_scaffold', 'helper_after']);
+    expect(result.nodes.some((node) => node.name === 'Ignored')).toBe(false);
+    expect(result.errors).toEqual([]);
+    expect(rewritten).toBe(source);
+  });
+
+  it('keeps the raw-string terminator at its original offset', () => {
+    const source = scaffoldSource('TAG');
+    const closer = source.indexOf(')TAG"');
+    const blanked = blankCppAnnotationMacroCalls(source);
+    expect(blanked.slice(closer, closer + 5)).toBe(')TAG"');
+    expect(blanked).toBe(source);
+  });
+
+  describe.each(annotationBlankers)('$name', ({ blank }) => {
+    it.each(['R', 'LR', 'u8R', 'uR', 'UR'])('leaves %s raw-string contents untouched', (prefix) => {
+      const source = `const auto* text = ${prefix}"TAG(
+DECLARE_THING(
+"quoted ) text" and 'characters' and a backslash \\
+)OTHER"
+value UPARAM(ref) UE_DEPRECATED(
+)TAG";
+`;
+      expect(blank(source)).toBe(source);
+    });
+
+    it.each(['', 'FIFTEEN_CHARS__', 'SIXTEEN_CHARS___'])('protects delimiter %j with CRLF', (delimiter) => {
+      const source = `const char* text = R"${delimiter}(\r\nUE_DEPRECATED(\r\n)${delimiter}";\r\n`;
+      expect(blank(source)).toBe(source);
+    });
+
+    it('leaves an unterminated raw string untouched', () => {
+      const source = 'const char* text = R"TAG(\nUE_DEPRECATED(1)\nint example;';
+      expect(blank(source)).toBe(source);
+    });
+  });
+
+  it.each([
+    { name: 'line-leading', blank: blankCppAnnotationMacroCalls, head: '', macro: 'ANNOTATE', tail: '\nint helper_after() { return 1; }\n' },
+    { name: 'inline', blank: blankCppInlineAnnotationMacros, head: 'using Alias ', macro: 'UE_DEPRECATED', tail: ' = int;\n' },
+    { name: 'C parameterized', blank: blankCParameterizedAnnotationMacros, head: 'static void ', macro: '__section', tail: ' helper_after(void) {}\n' },
+    { name: 'C iterator', blank: blankCStatementMacroCalls, head: 'void iterate() {\n  ', macro: 'for_each_item', tail: ' {\n    visit();\n  }\n}\n' },
+  ])('balances a genuine $name macro containing a raw-string argument', ({ blank, head, macro, tail }) => {
+    const annotation = `${macro}(R"TAG(" ) unbalanced ( " \\
+UE_DEPRECATED(
+)TAG")`;
+    expect(blank(head + annotation + tail))
+      .toBe(head + annotation.replace(/[^\r\n]/g, ' ') + tail);
+  });
+
+  it('balances a C declaration macro with a raw-string argument', () => {
+    const macro = 'static DECLARE_THING(R"TAG(" ) unbalanced ( ")TAG");';
+    const tail = '\nint helper_after(void) {}\n';
+    expect(blankCFileScopePrefixedDeclMacros(macro + tail))
+      .toBe(' '.repeat(macro.length) + tail);
+  });
+
+  it('preserves a raw argument while blanking a later C type-keyword argument', () => {
+    const literal = 'R"TAG(" ), struct Fake, ( ")TAG"';
+    const source = `take(${literal}, struct RealType);`;
+    expect(blankCTypeKeywordArgs(source)).toBe(`take(${literal},        RealType);`);
+  });
+
+  it('only counts designators outside raw arguments when blanking a C macro call', () => {
+    const literal = 'R"TAG(" ) .fake = 1 ( ")TAG"';
+    const head = 'void reset(void) {\n  RESET_THING(';
+    const tail = ');\n}\n';
+    expect(blankCDesignatedMacroArgs(head + literal + tail)).toBe(head + literal + tail);
+    const args = literal + ', .field = 1';
+    expect(blankCDesignatedMacroArgs(head + args + tail))
+      .toBe(head + ' '.repeat(args.length) + tail);
+  });
+
+  it.each([
+    { name: 'C iterator macros', blank: blankCStatementMacroCalls },
+    { name: 'C type arguments', blank: blankCTypeKeywordArgs },
+    { name: 'C declaration macros', blank: blankCFileScopePrefixedDeclMacros },
+    { name: 'C parameterized annotations', blank: blankCParameterizedAnnotationMacros },
+    { name: 'C designated initializer arguments', blank: blankCDesignatedMacroArgs },
+    { name: 'C preParse', blank: (source: string) => cExtractor.preParse!(source) },
+    { name: 'C++ preParse', blank: (source: string) => cppExtractor.preParse!(source, 'template.cpp') },
+  ])('$name leaves macro-like raw-string contents untouched', ({ blank }) => {
+    const source = `const auto* text = u8R"TAG(
+  for_each_item(item, list) {
+    visit(item);
+  }
+static DECLARE_THING(value);
+use(struct Example);
+  RESET_THING(.field = 1);
+class EXAMPLE_API Example {
+FORCEINLINE int example() {}
+};
+FMT_BEGIN_NAMESPACE
+int example;
+__section(
+)TAG";
+`;
+    expect(blank(source)).toBe(source);
+  });
+
+  it('ignores raw-string openers in comments and ordinary literals, then resumes blanking after a real raw string', () => {
+    const before = [
+      '// R"COMMENT(',
+      '/* LR"COMMENT( */',
+      'const char* quoted = "escaped R\\"STRING(";',
+      "const auto digit = 1'000;",
+      'const char quote = \'"\';',
+      scaffoldSource(),
+    ].join('\n');
+    const annotation = 'UPROPERTY(EditAnywhere)';
+    const tail = '\nint actual_field;\n';
+    expect(blankCppAnnotationMacroCalls(before + annotation + tail))
+      .toBe(before + ' '.repeat(annotation.length) + tail);
+  });
+});

src/extraction/languages/c-cpp.ts

@@ -480,6 +480,54 @@ export function blankMetalAttributes(source: string): string {
   return source.replace(METAL_ATTRIBUTE_RE, (m) => ' '.repeat(m.length));
 }
 
+/**
+ * Hide C++ raw literals from the offset-preserving preParse scans (#1505).
+ * Neither macro-shaped text in a raw body nor its `)delim"` closer is code.
+ * Mask the whole literal with non-whitespace, non-paren tokens, keeping line
+ * endings, so even line-based scanners treat a multiline raw argument as opaque.
+ * Restore untouched bytes afterward; a real enclosing annotation can still be
+ * removed in full. The pipeline masks once, and individual paren blankers also
+ * use this helper so they are safe when called directly.
+ */
+function maskCppRawStrings(source: string): { source: string; restore: (blanked: string) => string } {
+  const unchanged = { source, restore: (blanked: string): string => blanked };
+  if (source.indexOf('R"') === -1) return unchanged;
+  // Skip comments and ordinary literals before looking for a raw opener. The
+  // char-literal boundary leaves numeric digit separators (1'000) alone.
+  const re = /\/\/[^\r\n]*|\/\*[\s\S]*?(?:\*\/|$)|\b(?:u8|[LuU])?R"([^ \t\v\f\r\n()\\]{0,16})\(|"(?:\\[\s\S]|[^"\\])*(?:"|$)|(?<!\w)(?:u8|[LuU])?'(?:\\[\s\S]|[^'\\])*(?:'|$)/g;
+  const spans: Array<{ start: number; end: number }> = [];
+  const parts: string[] = [];
+  let last = 0;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(source)) !== null) {
+    if (m[1] === undefined) continue;
+    const closer = `)${m[1]}"`;
+    const close = source.indexOf(closer, re.lastIndex);
+    // An unterminated raw literal owns the rest of the file too.
+    const end = close < 0 ? source.length : close + closer.length;
+    spans.push({ start: m.index, end });
+    parts.push(source.slice(last, m.index), source.slice(m.index, end).replace(/[^\r\n]/g, '\0'));
+    last = re.lastIndex = end;
+  }
+  if (spans.length === 0) return unchanged;
+  parts.push(source.slice(last));
+  const masked = parts.join('');
+  return {
+    source: masked,
+    restore(blanked): string {
+      // A length-changing rewrite cannot be restored at the original offsets.
+      if (blanked === masked || blanked.length !== source.length) return source;
+      const chars = blanked.split('');
+      for (const { start, end } of spans) {
+        for (let i = start; i < end; i++) {
+          if (chars[i] === '\0') chars[i] = source[i] as string;
+        }
+      }
+      return chars.join('');
+    },
+  };
+}
+
 /**
  * Blank annotation-style macro invocations that decorate a declaration but carry
  * NO terminating semicolon — the pervasive Unreal-Engine reflection markup
@@ -519,6 +567,8 @@ export function blankMetalAttributes(source: string): string {
  */
 export function blankCppAnnotationMacroCalls(source: string): string {
   if (!/^[ \t]*[A-Z][A-Z0-9_]{2,}\s*\(/m.test(source)) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const chars = source.split('');
   const re = /^([ \t]*)([A-Z][A-Z0-9_]{2,})(\s*)\(/gm;
   let m: RegExpExecArray | null;
@@ -556,7 +606,7 @@ export function blankCppAnnotationMacroCalls(source: string): string {
     }
     re.lastIndex = end;
   }
-  return chars.join('');
+  return rawStrings.restore(chars.join(''));
 }
 
 /**
@@ -670,6 +720,8 @@ export function blankCppApiPrefixMacros(source: string): string {
 const CPP_INLINE_ANNOTATION_RE = /\b(?:UMETA|UPARAM|UE_DEPRECATED\w*)\s*\(/g;
 export function blankCppInlineAnnotationMacros(source: string): string {
   if (!/\b(?:UMETA|UPARAM|UE_DEPRECATED)/.test(source)) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const chars = source.split('');
   const re = new RegExp(CPP_INLINE_ANNOTATION_RE.source, 'g');
   let m: RegExpExecArray | null;
@@ -700,7 +752,7 @@ export function blankCppInlineAnnotationMacros(source: string): string {
     }
     re.lastIndex = end;
   }
-  return chars.join('');
+  return rawStrings.restore(chars.join(''));
 }
 
 /**
@@ -824,6 +876,8 @@ function restoreDirectiveLines(original: string, blanked: string): string {
  * or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving;
  * directive lines are restored at the end (see restoreDirectiveLines). */
 function preParseCppSource(source: string, filePath?: string): string {
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   // blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked
   // `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape
   // it matches (the _API token is already spaces by then).
@@ -842,7 +896,7 @@ function preParseCppSource(source: string, filePath?: string): string {
   } else if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
     blanked = blankCudaConstructs(blanked);
   }
-  return restoreDirectiveLines(source, blanked);
+  return rawStrings.restore(restoreDirectiveLines(source, blanked));
 }
 
 /**
@@ -970,6 +1024,8 @@ const C_STMT_MACRO_KEYWORDS = new Set([
   'if', 'while', 'for', 'switch', 'return', 'do', 'else', 'sizeof',
 ]);
 export function blankCStatementMacroCalls(source: string): string {
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const lines = source.split('\n');
   let changed = false;
   const content = (l: string): string => l.replace(/\r$/, '').trim();
@@ -1073,7 +1129,7 @@ export function blankCStatementMacroCalls(source: string): string {
     }
     changed = true;
   }
-  return changed ? lines.join('\n') : source;
+  return rawStrings.restore(changed ? lines.join('\n') : source);
 }
 
 /**
@@ -1244,8 +1300,8 @@ const C_PARAM_ANNOTATION_RE = new RegExp(
 );
 export function blankCParameterizedAnnotationMacros(source: string): string {
   if (source.indexOf('__') === -1) return source;
-  C_PARAM_ANNOTATION_RE.lastIndex = 0;
-  if (!C_PARAM_ANNOTATION_RE.test(source)) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   C_PARAM_ANNOTATION_RE.lastIndex = 0;
   let result = '';
   let last = 0;
@@ -1265,7 +1321,7 @@ export function blankCParameterizedAnnotationMacros(source: string): string {
     result += source.slice(last, start) + source.slice(start, end).replace(/[^\n\r]/g, ' ');
     last = end;
   }
-  return result + source.slice(last);
+  return rawStrings.restore(result + source.slice(last));
 }
 
 /**
@@ -1308,6 +1364,8 @@ const C_TYPE_ARG_OPENER_RE = /^(struct|union|enum)([ \t\r\n]+)([A-Za-z_]\w*)([ \
 const C_TYPE_ARG_SCAN_CAP = 600;
 export function blankCTypeKeywordArgs(source: string): string {
   if (!/\b(?:struct|union|enum)[ \t\r\n]/.test(source)) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   let chars: string[] | null = null;
   C_TYPE_ARG_HEAD_RE.lastIndex = 0;
   let m: RegExpExecArray | null;
@@ -1375,7 +1433,7 @@ export function blankCTypeKeywordArgs(source: string): string {
       atArgStart = false;
     }
   }
-  return chars ? chars.join('') : source;
+  return rawStrings.restore(chars ? chars.join('') : source);
 }
 
 /**
@@ -1403,6 +1461,8 @@ export function blankCTypeKeywordArgs(source: string): string {
 const C_PREFIXED_DECL_MACRO_RE = /^[ \t]*(?:static|extern)[ \t]+[A-Z][A-Z0-9_]{2,}[ \t]*\(/;
 export function blankCFileScopePrefixedDeclMacros(source: string): string {
   if (!/^[ \t]*(?:static|extern)[ \t]+[A-Z]/m.test(source)) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const lines = source.split('\n');
   let changed = false;
   for (let i = 0; i < lines.length; i++) {
@@ -1437,7 +1497,7 @@ export function blankCFileScopePrefixedDeclMacros(source: string): string {
     lines[i] = line.replace(/[^\n\r]/g, ' ');
     changed = true;
   }
-  return changed ? lines.join('\n') : source;
+  return rawStrings.restore(changed ? lines.join('\n') : source);
 }
 
 /**
@@ -1568,6 +1628,8 @@ export function blankCNamedVariadicDefineDots(source: string): string {
  */
 export function blankCDesignatedMacroArgs(source: string): string {
   if (source.indexOf('=') === -1) return source;
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const out = source.split('');
   const re = /^[ \t]*([A-Z_][A-Z0-9_]*)\s*\(/gm;
   let m: RegExpExecArray | null;
@@ -1589,10 +1651,12 @@ export function blankCDesignatedMacroArgs(source: string): string {
     for (let k = open + 1; k < close; k++) if (out[k] !== '\n') out[k] = ' ';
     re.lastIndex = close;
   }
-  return out.join('');
+  return rawStrings.restore(out.join(''));
 }
 
 function preParseCSource(source: string): string {
+  const rawStrings = maskCppRawStrings(source);
+  source = rawStrings.source;
   const inner = blankCDesignatedMacroArgs(blankCKernelAnnotations(blankCCplusplusGuardBodies(source)));
   let blanked = blankCLeadingAttrMacros(
     blankLoneMacroLines(
@@ -1618,7 +1682,7 @@ function preParseCSource(source: string): string {
   if (looksLikeCudaSource(blanked)) blanked = blankCudaConstructs(blanked);
   // The named-variadic `#define` pass runs AFTER the directive restore — it
   // deliberately edits directive lines (see its doc comment).
-  return blankCNamedVariadicDefineDots(restoreDirectiveLines(source, blanked));
+  return rawStrings.restore(blankCNamedVariadicDefineDots(restoreDirectiveLines(source, blanked)));
 }
 
 export const cppExtractor: LanguageExtractor = {