Browse Source

fix(scripts): enumerate repo files with a dirent walker instead of node fs.glob

Node's internal fs.glob, from some 24.x releases, lstat-probes
<matched>/<next segment> while expanding a ** pattern over a symlinked
file and throws ENOTDIR instead of skipping (observed on node 24.13.0
scanning snapshots/acp/image-compaction's symlinked
system-prompt.expected.md, which failed verify-md-wrap inside
check:ci:static). uniqueRepoFiles now walks with dirent types, never
probing a path under a file, and mirrors fs.glob's dot:false and
follow:false semantics: the four repo-files gate pattern sets enumerate
identically. Broken or cyclic symlinks fail loudly in the caller's
realpathSync, and pattern forms the walker does not model (metacharacters,
a trailing **, empty segments) are rejected up front. repo-files.spec.ts
pins the symlink, broken-link, and rejection behavior.
Chinesezjc 1 tuần trước cách đây
mục cha
commit
d79294f23a
2 tập tin đã thay đổi với 346 bổ sung4 xóa
  1. 176 0
      scripts/repo-files.spec.ts
  2. 170 4
      scripts/repo-files.ts

+ 176 - 0
scripts/repo-files.spec.ts

@@ -0,0 +1,176 @@
+import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, relative } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { uniqueRepoFiles } from './repo-files.ts'
+
+interface Tree {
+  root: string
+  clean: () => void
+}
+
+function makeTree(): Tree {
+  const parent = mkdtempSync(join(tmpdir(), 'repo-files-'))
+  const root = join(parent, 'root')
+  const outside = join(parent, 'outside') // reachable only through a symlinked dir
+  mkdirSync(join(root, 'a'), { recursive: true })
+  writeFileSync(join(root, 'a', 'snap.md'), 'real\n')
+  mkdirSync(join(root, 'd'), { recursive: true })
+  symlinkSync(join(root, 'a', 'snap.md'), join(root, 'd', 'snap.md'))
+  mkdirSync(outside, { recursive: true })
+  writeFileSync(join(outside, 'snap.md'), 'behind a symlinked dir\n')
+  symlinkSync(outside, join(root, 'linked-dir'))
+  mkdirSync(join(root, '.hidden'), { recursive: true })
+  writeFileSync(join(root, '.hidden', 'snap.md'), 'hidden by dot\n')
+  return { root, clean: () => { rmSync(parent, { recursive: true, force: true }) } }
+}
+
+describe('uniqueRepoFiles', () => {
+  it('enumerates ** matches without probing a symlinked file as a directory', () => {
+    const tree = makeTree()
+    try {
+      // Node's internal glob (from some 24.x releases) lstat-probes
+      // <symlink>/snap.md while expanding `**/snap.md` and throws ENOTDIR.
+      // The walker must return the real files, deduplicated by canonical
+      // target, without throwing on any node version.
+      const files = uniqueRepoFiles(tree.root, ['**/snap.md'])
+      const rootReal = realpathSync(tree.root)
+      const reals = files.map(file => relative(rootReal, file.real)).sort()
+      expect(reals).toEqual([join('a', 'snap.md')])
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('does not follow symlinked directories under ** or wildcard-match dot names', () => {
+    const tree = makeTree()
+    try {
+      const files = uniqueRepoFiles(tree.root, ['**/*.md'])
+      const paths = files.map(file => relative(tree.root, file.abs)).sort()
+      // The symlinked d/snap.md dedupes onto its a/snap.md target; the
+      // linked-dir and .hidden targets must not appear at all.
+      expect(paths).toEqual([join('a', 'snap.md')])
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('follows a literal segment that names a symlinked directory', () => {
+    const tree = makeTree()
+    try {
+      // Node glob resolves literal segments with stat, so `linked-dir/**` and
+      // `linked-dir/*` enter the symlinked directory's target; `**` and
+      // wildcard segments resolve with dirent types and do not.
+      const files = uniqueRepoFiles(tree.root, ['linked-dir/**/*.md'])
+      expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('linked-dir', 'snap.md')])
+      expect(uniqueRepoFiles(tree.root, ['linked-dir/*.md']).map(file => relative(tree.root, file.abs)))
+        .toEqual([join('linked-dir', 'snap.md')])
+      // A wildcard first segment never enters the symlinked directory.
+      expect(uniqueRepoFiles(tree.root, ['*/snap.md']).map(file => relative(tree.root, file.abs)))
+        .toEqual([join('a', 'snap.md')])
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('follows repeated literal symlinked directories like node glob and terminates', () => {
+    const parent = mkdtempSync(join(tmpdir(), 'repo-files-cycle-'))
+    try {
+      const root = join(parent, 'root')
+      mkdirSync(join(root, 'a'), { recursive: true })
+      writeFileSync(join(root, 'a', 'snap.md'), 'x\n')
+      symlinkSync(root, join(root, 'cyc'))
+      // Each literal `cyc` segment resolves through stat and enters the
+      // symlinked directory again, exactly as node glob does for a repeated
+      // literal; recursion stays bounded because each literal consumes one
+      // pattern segment and `**` only enters real directories.
+      expect(uniqueRepoFiles(root, ['cyc/**/snap.md']).map(file => relative(root, file.abs)))
+        .toEqual([join('cyc', 'a', 'snap.md')])
+      expect(uniqueRepoFiles(root, ['cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
+        .toEqual([join('cyc', 'cyc', 'a', 'snap.md')])
+      expect(uniqueRepoFiles(root, ['cyc/cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
+        .toEqual([join('cyc', 'cyc', 'cyc', 'a', 'snap.md')])
+    } finally {
+      rmSync(parent, { recursive: true, force: true })
+    }
+  })
+
+  it('reports a matched file canonical target for downstream existence checks', () => {
+    const tree = makeTree()
+    try {
+      const files = uniqueRepoFiles(tree.root, ['a/*.md'])
+      expect(files).toHaveLength(1)
+      expect(files[0]!.real).toBe(join(realpathSync(tree.root), 'a', 'snap.md'))
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('fails loudly on a broken symlink instead of shrinking the corpus', () => {
+    const tree = makeTree()
+    try {
+      // A healthy tree with symlinked files must not throw.
+      expect(() => uniqueRepoFiles(tree.root, ['a/*.md'])).not.toThrow()
+    } finally {
+      tree.clean()
+    }
+    const parent = mkdtempSync(join(tmpdir(), 'repo-files-broken-'))
+    try {
+      const root = join(parent, 'root')
+      mkdirSync(root, { recursive: true })
+      writeFileSync(join(root, 'gone.md'), 'x\n')
+      symlinkSync(join(root, 'gone.md'), join(root, 'broken-link.md'))
+      rmSync(join(root, 'gone.md'))
+      // realpathSync on the matched broken link must throw, exactly as it did
+      // under the node-glob implementation, instead of silently shrinking the
+      // scanned corpus.
+      expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow()
+      // A broken symlink under a literal non-final segment matches nothing,
+      // again as node glob silently returns no match for it.
+      symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir'))
+      expect(uniqueRepoFiles(root, ['broken-dir/*.md'])).toEqual([])
+    } finally {
+      rmSync(parent, { recursive: true, force: true })
+    }
+  })
+
+  it('rejects glob syntax the walker does not model instead of matching nothing', () => {
+    const tree = makeTree()
+    try {
+      // Node glob would interpret the bracket class; the walker must fail
+      // loudly rather than expand the pattern as a literal and quietly match
+      // nothing.
+      expect(() => uniqueRepoFiles(tree.root, ['**/*.[cm]d'])).toThrow(/does not model glob syntax/)
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('rejects a trailing ** segment instead of silently returning nothing', () => {
+    const tree = makeTree()
+    try {
+      expect(() => uniqueRepoFiles(tree.root, ['a/**'])).toThrow(/trailing \*\*/)
+      // A trailing slash yields an empty final segment that can never match.
+      expect(() => uniqueRepoFiles(tree.root, ['a/**/'])).toThrow(/empty segments/)
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('folds a . segment and rejects a .. segment like node glob semantics', () => {
+    const tree = makeTree()
+    try {
+      // Node glob normalizes a `.` segment away, so `./a/snap.md` matches
+      // a/snap.md instead of looking for a directory named `.`.
+      expect(uniqueRepoFiles(tree.root, ['./a/snap.md']).map(file => relative(tree.root, file.abs)))
+        .toEqual([join('a', 'snap.md')])
+      expect(uniqueRepoFiles(tree.root, ['a/./snap.md']).map(file => relative(tree.root, file.abs)))
+        .toEqual([join('a', 'snap.md')])
+      // A `..` segment escapes the scanned root; the walker must fail loudly
+      // rather than silently match nothing.
+      expect(() => uniqueRepoFiles(tree.root, ['a/../snap.md'])).toThrow(/does not model \.\. segments/)
+    } finally {
+      tree.clean()
+    }
+  })
+})

+ 170 - 4
scripts/repo-files.ts

@@ -1,7 +1,7 @@
 /** Shared repository file discovery and line-oriented reference scanning. */
 
-import { globSync, readFileSync, realpathSync } from 'node:fs'
-import { relative, resolve, sep } from 'node:path'
+import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
+import { join, relative, resolve, sep } from 'node:path'
 
 /** One authored path plus its canonical target for symlink deduplication. */
 export interface RepoFile {
@@ -26,6 +26,173 @@ export function isArchivedAgentNotePath(path: string): boolean {
   return path.replaceAll('\\', '/').startsWith('.agents/notes/archived/')
 }
 
+/**
+ * Whether a pattern segment matches a directory or file name. Supports `*` and
+ * `?` inside a segment and mirrors node's glob `dot: false`: a segment whose
+ * first character is a wildcard does not match dot names. `**` is handled as a
+ * whole segment by the walker, never here.
+ */
+function segmentMatches(pattern: string, name: string): boolean {
+  if (name.startsWith('.') && (pattern.startsWith('*') || pattern.startsWith('?'))) return false
+  let expression = ''
+  for (const character of pattern) {
+    if (character === '*') expression += '.*'
+    else if (character === '?') expression += '.'
+    else expression += character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  }
+  return new RegExp(`^${expression}$`).test(name)
+}
+
+/**
+ * Reject glob syntax the walker does not model. Node's `fs.glob` understands
+ * character classes, brace alternation, and extglobs; expanding those
+ * silently as literals would match nothing and quietly shrink a gate's
+ * corpus, so a pattern segment using them fails loudly instead. Pure literal
+ * segments must therefore contain none of the rejected metacharacters
+ * either, even where node glob would read them literally.
+ */
+const UNSUPPORTED_GLOB = /[[\]{}()!+@\\]/
+
+function assertSupportedSegment(segment: string): void {
+  if (UNSUPPORTED_GLOB.test(segment)) {
+    throw new TypeError(`repo-files walker does not model glob syntax in segment: ${segment}`)
+  }
+}
+
+/**
+ * Walk `root` matching one repository-relative glob without node's `fs.glob`.
+ * The repository's own walker exists because node's internal glob, from some
+ * 24.x releases, lstat-probes `<matched>/<next segment>` for symlinked files
+ * while expanding `**` and throws ENOTDIR instead of skipping (observed on
+ * node 24.13.0 scanning `snapshots/acp/image-compaction`'s symlinked
+ * `system-prompt.expected.md`). The walker decides directoryhood from dirent
+ * types and stat results, never by probing a path under a file, so the same
+ * tree enumerates identically on every node version.
+ *
+ * Segment semantics: literal segments resolve through `stat`, so a literal
+ * naming a symlinked directory enters that directory exactly as node glob
+ * does — including a literal repeated across segments, which node glob
+ * resolves each time; wildcard segments and `**` resolve through dirent
+ * types, so they never enter symlinked directories and never match dot
+ * names. Node documents `follow: false` only for `**` expansion, so the
+ * wildcard-side behavior is this walker's own contract, pinned by
+ * repo-files.spec.ts, rather than a cross-version node guarantee; the four
+ * consuming gates' current patterns contain no wildcard segment over a
+ * symlinked directory, so the corpus is unchanged. `**` spans zero or more
+ * directories. The final segment matches files and symlinks; a broken or
+ * cyclic symlink then fails loudly in the caller's realpathSync rather than
+ * shrinking the scanned corpus, while a broken symlink under a literal
+ * non-final segment matches nothing, again as node glob does. Traversal
+ * terminates without a visited set: each literal segment consumes one pattern
+ * segment per recursion, and `**` recurses only into real directories, which
+ * form a finite tree because symlinked directories are never expanded by it.
+ * Pattern segments support `*`, `?`, and literals that contain none of
+ * `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and
+ * empty segments, is rejected loudly up front, while a `.` segment is folded
+ * away exactly as node glob normalizes it. Returns repository-relative slash
+ * paths, sorted.
+ */
+function expandGlob(root: string, pattern: string): string[] {
+  const segments: string[] = []
+  for (const segment of pattern.split('/')) {
+    if (segment === '.') {
+      // Node glob normalizes a `.` segment away; dropping it here keeps
+      // `./README.md` and `a/./b` matching exactly as node glob does instead
+      // of silently matching nothing.
+      continue
+    }
+    if (segment === '..') {
+      // A `..` segment escapes the scanned root and interacts with `**` in
+      // ways node glob special-cases; no gate pattern uses one, so the walker
+      // rejects the form loudly instead of silently matching nothing.
+      throw new TypeError(`repo-files walker does not model .. segments in pattern: ${pattern}`)
+    }
+    if (segment === '') {
+      // A leading, doubled, or trailing slash yields an empty segment that
+      // can never match an entry; node glob would tolerate the form, so the
+      // walker must reject it loudly instead of silently returning nothing.
+      throw new TypeError(`repo-files walker does not model empty segments in pattern: ${pattern}`)
+    }
+    segments.push(segment)
+  }
+  if (segments.length === 0) {
+    throw new TypeError(`repo-files walker does not model a pattern with no segments: ${pattern}`)
+  }
+  for (const segment of segments) {
+    if (segment !== '**') assertSupportedSegment(segment)
+  }
+  // A trailing `**` would match files, directories, and symlinks below the
+  // prefix; the walker models `**` only as a directory-spanning segment, so
+  // it must reject the form loudly instead of silently returning nothing.
+  if (segments[segments.length - 1] === '**') {
+    throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`)
+  }
+  const out: string[] = []
+
+  const visit = (dirAbs: string, dirRel: string, index: number): void => {
+    if (index >= segments.length) return
+    if (segments[index] !== '**') {
+      visitSegment(dirAbs, dirRel, index)
+      return
+    }
+    // `**` consumes zero directories here and one directory per recursion.
+    visitSegment(dirAbs, dirRel, index + 1)
+    for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
+      if (entry.name.startsWith('.') || !entry.isDirectory()) continue
+      visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index)
+    }
+  }
+
+  const visitSegment = (dirAbs: string, dirRel: string, index: number): void => {
+    if (index >= segments.length) return
+    const segment = segments[index]
+    if (segment === undefined) return
+    if (segment === '**') {
+      visit(dirAbs, dirRel, index)
+      return
+    }
+    const last = index === segments.length - 1
+    for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
+      if (!segmentMatches(segment, entry.name)) continue
+      const childAbs = join(dirAbs, entry.name)
+      const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`
+      if (last) {
+        // Files and symlinks match; a broken or cyclic symlink then fails
+        // loudly in the caller's realpathSync exactly as node glob did,
+        // rather than silently shrinking the scanned corpus.
+        if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel)
+        continue
+      }
+      if (entry.isDirectory()) {
+        visitSegment(childAbs, childRel, index + 1)
+        continue
+      }
+      // A literal non-final segment resolves through stat like node glob's,
+      // so it enters a symlinked directory; wildcard segments never reach
+      // this branch because they are matched from dirent types above.
+      if (!entry.isSymbolicLink() || hasWildcard(segment)) continue
+      let target: ReturnType<typeof statSync>
+      try {
+        target = statSync(childAbs)
+      } catch {
+        // A broken symlink matches nothing under a literal segment, as node
+        // glob silently returns no match for it.
+        continue
+      }
+      if (!target.isDirectory()) continue
+      visitSegment(childAbs, childRel, index + 1)
+    }
+  }
+
+  visit(root, '.', 0)
+  return out.sort()
+}
+
+/** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */
+function hasWildcard(segment: string): boolean {
+  return segment.includes('*') || segment.includes('?')
+}
+
 /**
  * Expand repository-relative globs and deduplicate symlinked files.
  * @param root - absolute repository root.
@@ -41,8 +208,7 @@ export function uniqueRepoFiles(
   const seen = new Set<string>()
   const files: RepoFile[] = []
   for (const pattern of patterns) {
-    for (const match of globSync(pattern, { cwd: root })) {
-      const repoPath = match.split(sep).join('/')
+    for (const repoPath of expandGlob(root, pattern)) {
       if (isExcluded(repoPath)) continue
       const abs = resolve(root, repoPath)
       const real = realpathSync(abs)