|
|
@@ -14,14 +14,13 @@ import {
|
|
|
FileRecord,
|
|
|
ExtractionResult,
|
|
|
ExtractionError,
|
|
|
- CodeGraphConfig,
|
|
|
} from '../types';
|
|
|
import { QueryBuilder } from '../db/queries';
|
|
|
import { extractFromSource } from './tree-sitter';
|
|
|
-import { detectLanguage, isLanguageSupported, initGrammars, loadGrammarsForLanguages } from './grammars';
|
|
|
+import { detectLanguage, isSourceFile, isLanguageSupported, initGrammars, loadGrammarsForLanguages } from './grammars';
|
|
|
import { logDebug, logWarn } from '../errors';
|
|
|
import { validatePathWithinRoot, normalizePath } from '../utils';
|
|
|
-import picomatch from 'picomatch';
|
|
|
+import ignore, { Ignore } from 'ignore';
|
|
|
import { detectFrameworks } from '../resolution/frameworks';
|
|
|
import type { ResolutionContext } from '../resolution/types';
|
|
|
|
|
|
@@ -94,36 +93,11 @@ export function hashContent(content: string): string {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * Check if a path matches any glob pattern (simplified)
|
|
|
+ * Skip files larger than this (bytes). Generated bundles, minified JS, and
|
|
|
+ * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
|
|
|
+ * symbols. 1 MB covers essentially all hand-written source.
|
|
|
*/
|
|
|
-function matchesGlob(filePath: string, pattern: string): boolean {
|
|
|
- filePath = normalizePath(filePath);
|
|
|
- return picomatch.isMatch(filePath, pattern, { dot: true });
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * Check if a file should be included based on config
|
|
|
- */
|
|
|
-export function shouldIncludeFile(
|
|
|
- filePath: string,
|
|
|
- config: CodeGraphConfig
|
|
|
-): boolean {
|
|
|
- // Check exclude patterns first
|
|
|
- for (const pattern of config.exclude) {
|
|
|
- if (matchesGlob(filePath, pattern)) {
|
|
|
- return false;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // Check include patterns
|
|
|
- for (const pattern of config.include) {
|
|
|
- if (matchesGlob(filePath, pattern)) {
|
|
|
- return true;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return false;
|
|
|
-}
|
|
|
+const MAX_FILE_SIZE = 1024 * 1024;
|
|
|
|
|
|
/**
|
|
|
* Collect git-visible files (tracked + untracked, .gitignore-respected) from the
|
|
|
@@ -230,7 +204,7 @@ interface GitChanges {
|
|
|
* Use `git status` to detect changed files instead of scanning every file.
|
|
|
* Returns null on failure so callers fall back to full scan.
|
|
|
*/
|
|
|
-function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChanges | null {
|
|
|
+function getGitChangedFiles(rootDir: string): GitChanges | null {
|
|
|
try {
|
|
|
const output = execFileSync(
|
|
|
'git',
|
|
|
@@ -248,8 +222,8 @@ function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChange
|
|
|
const statusCode = line.substring(0, 2);
|
|
|
const filePath = normalizePath(line.substring(3));
|
|
|
|
|
|
- // Skip files that don't match include/exclude config
|
|
|
- if (!shouldIncludeFile(filePath, config)) continue;
|
|
|
+ // Skip non-source files (git status already omits .gitignored paths).
|
|
|
+ if (!isSourceFile(filePath)) continue;
|
|
|
|
|
|
if (statusCode === '??') {
|
|
|
added.push(filePath);
|
|
|
@@ -268,20 +242,14 @@ function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChange
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * Marker file name that indicates a directory (and all children) should be skipped
|
|
|
- */
|
|
|
-const CODEGRAPH_IGNORE_MARKER = '.codegraphignore';
|
|
|
-
|
|
|
-/**
|
|
|
- * Recursively scan directory for source files.
|
|
|
+ * Recursively scan a directory for source files.
|
|
|
*
|
|
|
- * In git repos, uses `git ls-files` to get the file list (inherently
|
|
|
- * respects .gitignore at all levels), then filters by config include patterns.
|
|
|
- * Falls back to filesystem walk for non-git projects.
|
|
|
+ * In git repos, uses `git ls-files` (inherently respects .gitignore at all
|
|
|
+ * levels), then keeps files with a supported source extension. For non-git
|
|
|
+ * projects, falls back to a filesystem walk that parses .gitignore itself.
|
|
|
*/
|
|
|
export function scanDirectory(
|
|
|
rootDir: string,
|
|
|
- config: CodeGraphConfig,
|
|
|
onProgress?: (current: number, file: string) => void
|
|
|
): string[] {
|
|
|
// Fast path: use git to get all visible files (respects .gitignore everywhere)
|
|
|
@@ -290,7 +258,7 @@ export function scanDirectory(
|
|
|
const files: string[] = [];
|
|
|
let count = 0;
|
|
|
for (const filePath of gitFiles) {
|
|
|
- if (shouldIncludeFile(filePath, config)) {
|
|
|
+ if (isSourceFile(filePath)) {
|
|
|
files.push(filePath);
|
|
|
count++;
|
|
|
onProgress?.(count, filePath);
|
|
|
@@ -300,7 +268,7 @@ export function scanDirectory(
|
|
|
}
|
|
|
|
|
|
// Fallback: walk filesystem for non-git projects
|
|
|
- return scanDirectoryWalk(rootDir, config, onProgress);
|
|
|
+ return scanDirectoryWalk(rootDir, onProgress);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -309,7 +277,6 @@ export function scanDirectory(
|
|
|
*/
|
|
|
export async function scanDirectoryAsync(
|
|
|
rootDir: string,
|
|
|
- config: CodeGraphConfig,
|
|
|
onProgress?: (current: number, file: string) => void
|
|
|
): Promise<string[]> {
|
|
|
const gitFiles = getGitVisibleFiles(rootDir);
|
|
|
@@ -317,7 +284,7 @@ export async function scanDirectoryAsync(
|
|
|
const files: string[] = [];
|
|
|
let count = 0;
|
|
|
for (const filePath of gitFiles) {
|
|
|
- if (shouldIncludeFile(filePath, config)) {
|
|
|
+ if (isSourceFile(filePath)) {
|
|
|
files.push(filePath);
|
|
|
count++;
|
|
|
onProgress?.(count, filePath);
|
|
|
@@ -330,7 +297,7 @@ export async function scanDirectoryAsync(
|
|
|
return files;
|
|
|
}
|
|
|
|
|
|
- return scanDirectoryWalk(rootDir, config, onProgress);
|
|
|
+ return scanDirectoryWalk(rootDir, onProgress);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -338,14 +305,44 @@ export async function scanDirectoryAsync(
|
|
|
*/
|
|
|
function scanDirectoryWalk(
|
|
|
rootDir: string,
|
|
|
- config: CodeGraphConfig,
|
|
|
onProgress?: (current: number, file: string) => void
|
|
|
): string[] {
|
|
|
const files: string[] = [];
|
|
|
let count = 0;
|
|
|
const visitedDirs = new Set<string>();
|
|
|
|
|
|
- function walk(dir: string): void {
|
|
|
+ // A .gitignore matcher scoped to the directory that declared it. Patterns in
|
|
|
+ // a nested .gitignore are relative to that directory, so we keep the dir
|
|
|
+ // alongside the matcher and test paths relative to it — mirroring how git
|
|
|
+ // applies .gitignore files at every level.
|
|
|
+ interface ScopedIgnore {
|
|
|
+ dir: string;
|
|
|
+ ig: Ignore;
|
|
|
+ }
|
|
|
+
|
|
|
+ const loadIgnore = (dir: string): ScopedIgnore | null => {
|
|
|
+ try {
|
|
|
+ const giPath = path.join(dir, '.gitignore');
|
|
|
+ if (fs.existsSync(giPath)) {
|
|
|
+ return { dir, ig: ignore().add(fs.readFileSync(giPath, 'utf-8')) };
|
|
|
+ }
|
|
|
+ } catch {
|
|
|
+ // Unreadable .gitignore — treat as absent.
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ };
|
|
|
+
|
|
|
+ const isIgnored = (fullPath: string, isDir: boolean, matchers: ScopedIgnore[]): boolean => {
|
|
|
+ for (const { dir, ig } of matchers) {
|
|
|
+ let rel = normalizePath(path.relative(dir, fullPath));
|
|
|
+ if (!rel || rel.startsWith('..')) continue; // not under this matcher's dir
|
|
|
+ if (isDir) rel += '/'; // dir-only rules (e.g. `build/`) only match with the slash
|
|
|
+ if (ig.ignores(rel)) return true;
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ };
|
|
|
+
|
|
|
+ function walk(dir: string, matchers: ScopedIgnore[]): void {
|
|
|
let realDir: string;
|
|
|
try {
|
|
|
realDir = fs.realpathSync(dir);
|
|
|
@@ -360,12 +357,9 @@ function scanDirectoryWalk(
|
|
|
}
|
|
|
visitedDirs.add(realDir);
|
|
|
|
|
|
- // Check for .codegraphignore marker file
|
|
|
- const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER);
|
|
|
- if (fs.existsSync(ignoreMarker)) {
|
|
|
- logDebug('Skipping directory due to .codegraphignore marker', { dir });
|
|
|
- return;
|
|
|
- }
|
|
|
+ // This directory's own .gitignore (if present) applies to everything below it.
|
|
|
+ const own = loadIgnore(dir);
|
|
|
+ const active = own ? [...matchers, own] : matchers;
|
|
|
|
|
|
let entries: fs.Dirent[];
|
|
|
try {
|
|
|
@@ -376,6 +370,9 @@ function scanDirectoryWalk(
|
|
|
}
|
|
|
|
|
|
for (const entry of entries) {
|
|
|
+ // Never descend into git internals or our own data directory.
|
|
|
+ if (entry.name === '.git' || entry.name === '.codegraph') continue;
|
|
|
+
|
|
|
const fullPath = path.join(dir, entry.name);
|
|
|
const relativePath = normalizePath(path.relative(rootDir, fullPath));
|
|
|
|
|
|
@@ -384,19 +381,11 @@ function scanDirectoryWalk(
|
|
|
const realTarget = fs.realpathSync(fullPath);
|
|
|
const stat = fs.statSync(realTarget);
|
|
|
if (stat.isDirectory()) {
|
|
|
- const dirPattern = relativePath + '/';
|
|
|
- let excluded = false;
|
|
|
- for (const pattern of config.exclude) {
|
|
|
- if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
|
|
|
- excluded = true;
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- if (!excluded) {
|
|
|
- walk(fullPath);
|
|
|
+ if (!isIgnored(fullPath, true, active)) {
|
|
|
+ walk(fullPath, active);
|
|
|
}
|
|
|
} else if (stat.isFile()) {
|
|
|
- if (shouldIncludeFile(relativePath, config)) {
|
|
|
+ if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath)) {
|
|
|
files.push(relativePath);
|
|
|
count++;
|
|
|
onProgress?.(count, relativePath);
|
|
|
@@ -409,19 +398,11 @@ function scanDirectoryWalk(
|
|
|
}
|
|
|
|
|
|
if (entry.isDirectory()) {
|
|
|
- const dirPattern = relativePath + '/';
|
|
|
- let excluded = false;
|
|
|
- for (const pattern of config.exclude) {
|
|
|
- if (matchesGlob(dirPattern, pattern) || matchesGlob(relativePath, pattern)) {
|
|
|
- excluded = true;
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- if (!excluded) {
|
|
|
- walk(fullPath);
|
|
|
+ if (!isIgnored(fullPath, true, active)) {
|
|
|
+ walk(fullPath, active);
|
|
|
}
|
|
|
} else if (entry.isFile()) {
|
|
|
- if (shouldIncludeFile(relativePath, config)) {
|
|
|
+ if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath)) {
|
|
|
files.push(relativePath);
|
|
|
count++;
|
|
|
onProgress?.(count, relativePath);
|
|
|
@@ -430,7 +411,7 @@ function scanDirectoryWalk(
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- walk(rootDir);
|
|
|
+ walk(rootDir, []);
|
|
|
return files;
|
|
|
}
|
|
|
|
|
|
@@ -439,7 +420,6 @@ function scanDirectoryWalk(
|
|
|
*/
|
|
|
export class ExtractionOrchestrator {
|
|
|
private rootDir: string;
|
|
|
- private config: CodeGraphConfig;
|
|
|
private queries: QueryBuilder;
|
|
|
/**
|
|
|
* Names of frameworks detected for this project, populated by indexAll().
|
|
|
@@ -449,9 +429,8 @@ export class ExtractionOrchestrator {
|
|
|
*/
|
|
|
private detectedFrameworkNames: string[] | null = null;
|
|
|
|
|
|
- constructor(rootDir: string, config: CodeGraphConfig, queries: QueryBuilder) {
|
|
|
+ constructor(rootDir: string, queries: QueryBuilder) {
|
|
|
this.rootDir = rootDir;
|
|
|
- this.config = config;
|
|
|
this.queries = queries;
|
|
|
}
|
|
|
|
|
|
@@ -500,7 +479,7 @@ export class ExtractionOrchestrator {
|
|
|
*/
|
|
|
private ensureDetectedFrameworks(files?: string[]): string[] {
|
|
|
if (this.detectedFrameworkNames !== null) return this.detectedFrameworkNames;
|
|
|
- const fileList = files ?? scanDirectory(this.rootDir, this.config);
|
|
|
+ const fileList = files ?? scanDirectory(this.rootDir);
|
|
|
const context = this.buildDetectionContext(fileList);
|
|
|
this.detectedFrameworkNames = detectFrameworks(context).map((r) => r.name);
|
|
|
return this.detectedFrameworkNames;
|
|
|
@@ -534,7 +513,7 @@ export class ExtractionOrchestrator {
|
|
|
total: 0,
|
|
|
});
|
|
|
|
|
|
- const files = await scanDirectoryAsync(this.rootDir, this.config, (current, file) => {
|
|
|
+ const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
|
|
|
onProgress?.({
|
|
|
phase: 'scanning',
|
|
|
current,
|
|
|
@@ -802,18 +781,16 @@ export class ExtractionOrchestrator {
|
|
|
continue;
|
|
|
}
|
|
|
|
|
|
- // Honour config.maxFileSize. Without this check, vendored
|
|
|
- // generated headers, minified bundles, and other multi-MB
|
|
|
- // files get indexed despite the user setting a size cap —
|
|
|
- // wasting WASM heap and the worker recycle budget on inputs
|
|
|
- // the user explicitly opted out of. The single-file extractFile
|
|
|
- // path already enforces this; the bulk path used to silently
|
|
|
- // skip the check.
|
|
|
- if (stats.size > this.config.maxFileSize) {
|
|
|
+ // Honour MAX_FILE_SIZE. Without this check, vendored generated
|
|
|
+ // headers, minified bundles, and other multi-MB files get indexed,
|
|
|
+ // wasting WASM heap and the worker recycle budget on inputs with no
|
|
|
+ // useful symbols. The single-file extractFile path already enforces
|
|
|
+ // this; the bulk path used to silently skip the check.
|
|
|
+ if (stats.size > MAX_FILE_SIZE) {
|
|
|
processed++;
|
|
|
filesSkipped++;
|
|
|
errors.push({
|
|
|
- message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
|
|
|
+ message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
|
|
|
filePath,
|
|
|
severity: 'warning',
|
|
|
code: 'size_exceeded',
|
|
|
@@ -1108,14 +1085,14 @@ export class ExtractionOrchestrator {
|
|
|
}
|
|
|
|
|
|
// Check file size
|
|
|
- if (stats.size > this.config.maxFileSize) {
|
|
|
+ if (stats.size > MAX_FILE_SIZE) {
|
|
|
return {
|
|
|
nodes: [],
|
|
|
edges: [],
|
|
|
unresolvedReferences: [],
|
|
|
errors: [
|
|
|
{
|
|
|
- message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`,
|
|
|
+ message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
|
|
|
filePath: relativePath,
|
|
|
severity: 'warning',
|
|
|
code: 'size_exceeded',
|
|
|
@@ -1245,7 +1222,7 @@ export class ExtractionOrchestrator {
|
|
|
});
|
|
|
|
|
|
const filesToIndex: string[] = [];
|
|
|
- const gitChanges = getGitChangedFiles(this.rootDir, this.config);
|
|
|
+ const gitChanges = getGitChangedFiles(this.rootDir);
|
|
|
|
|
|
if (gitChanges) {
|
|
|
// === Git fast path ===
|
|
|
@@ -1291,7 +1268,7 @@ export class ExtractionOrchestrator {
|
|
|
}
|
|
|
} else {
|
|
|
// === Fallback: full scan (non-git project or git failure) ===
|
|
|
- const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
|
|
|
+ const currentFiles = new Set(scanDirectory(this.rootDir));
|
|
|
filesChecked = currentFiles.size;
|
|
|
|
|
|
// Build Map for O(1) lookups instead of .find() per file
|
|
|
@@ -1376,7 +1353,7 @@ export class ExtractionOrchestrator {
|
|
|
* Uses git status as a fast path when available, falling back to full scan.
|
|
|
*/
|
|
|
getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
|
|
|
- const gitChanges = getGitChangedFiles(this.rootDir, this.config);
|
|
|
+ const gitChanges = getGitChangedFiles(this.rootDir);
|
|
|
|
|
|
if (gitChanges) {
|
|
|
// === Git fast path ===
|
|
|
@@ -1420,7 +1397,7 @@ export class ExtractionOrchestrator {
|
|
|
}
|
|
|
|
|
|
// === Fallback: full scan (non-git project or git failure) ===
|
|
|
- const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
|
|
|
+ const currentFiles = new Set(scanDirectory(this.rootDir));
|
|
|
const trackedFiles = this.queries.getAllFiles();
|
|
|
|
|
|
// Build Map for O(1) lookups
|
|
|
@@ -1467,4 +1444,4 @@ export class ExtractionOrchestrator {
|
|
|
|
|
|
// Re-export useful types and functions
|
|
|
export { extractFromSource } from './tree-sitter';
|
|
|
-export { detectLanguage, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './grammars';
|
|
|
+export { detectLanguage, isSourceFile, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './grammars';
|
Colby Mchenry