|
@@ -99,6 +99,16 @@ export interface IndexResult {
|
|
|
* counts. Only set by full-index runs (indexAll), not indexFiles/sync.
|
|
* counts. Only set by full-index runs (indexAll), not indexFiles/sync.
|
|
|
*/
|
|
*/
|
|
|
filesDiscovered?: number;
|
|
filesDiscovered?: number;
|
|
|
|
|
+ /**
|
|
|
|
|
+ * Files the scan saw but has no grammar for, tallied by extension. Only the
|
|
|
|
|
+ * degenerate case needs it: a project of unsupported files otherwise looks
|
|
|
|
|
+ * exactly like an empty one (0 files, state `complete`), so nothing tells the
|
|
|
|
|
+ * user — or an agent — that there was code here CodeGraph could not read
|
|
|
|
|
+ * (#1502). Counted during the scan's existing walk.
|
|
|
|
|
+ */
|
|
|
|
|
+ filesSkippedUnsupported?: number;
|
|
|
|
|
+ /** The most common unsupported extensions, biggest first. */
|
|
|
|
|
+ topUnsupportedExtensions?: { ext: string; count: number }[];
|
|
|
nodesCreated: number;
|
|
nodesCreated: number;
|
|
|
edgesCreated: number;
|
|
edgesCreated: number;
|
|
|
errors: ExtractionError[];
|
|
errors: ExtractionError[];
|
|
@@ -1408,9 +1418,30 @@ export function scanDirectory(
|
|
|
* Async variant of scanDirectory that yields to the event loop periodically,
|
|
* Async variant of scanDirectory that yields to the event loop periodically,
|
|
|
* allowing worker threads to receive and render progress messages.
|
|
* allowing worker threads to receive and render progress messages.
|
|
|
*/
|
|
*/
|
|
|
|
|
+/**
|
|
|
|
|
+ * What a scan saw but could not index, tallied by extension.
|
|
|
|
|
+ *
|
|
|
|
|
+ * Filled during the walk the scan already performs — a project of unsupported
|
|
|
|
|
+ * files is otherwise indistinguishable from an empty one, because unsupported
|
|
|
|
|
+ * extensions are filtered out at discovery and never counted anywhere (#1502).
|
|
|
|
|
+ */
|
|
|
|
|
+export interface ScanSkipStats {
|
|
|
|
|
+ /** Lowercased extension (with dot) → how many files carried it. */
|
|
|
|
|
+ unsupportedByExtension: Map<string, number>;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/** Record one file the scan declined to index. */
|
|
|
|
|
+function tallySkip(stats: ScanSkipStats | undefined, rel: string): void {
|
|
|
|
|
+ if (!stats) return;
|
|
|
|
|
+ const ext = path.extname(rel).toLowerCase();
|
|
|
|
|
+ if (!ext) return;
|
|
|
|
|
+ stats.unsupportedByExtension.set(ext, (stats.unsupportedByExtension.get(ext) ?? 0) + 1);
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
export async function scanDirectoryAsync(
|
|
export async function scanDirectoryAsync(
|
|
|
rootDir: string,
|
|
rootDir: string,
|
|
|
- onProgress?: (current: number, file: string) => void
|
|
|
|
|
|
|
+ onProgress?: (current: number, file: string) => void,
|
|
|
|
|
+ stats?: ScanSkipStats
|
|
|
): Promise<string[]> {
|
|
): Promise<string[]> {
|
|
|
// Custom extension → language overrides from the project's codegraph.json.
|
|
// Custom extension → language overrides from the project's codegraph.json.
|
|
|
const overrides = loadExtensionOverrides(rootDir);
|
|
const overrides = loadExtensionOverrides(rootDir);
|
|
@@ -1428,12 +1459,14 @@ export async function scanDirectoryAsync(
|
|
|
if (count % 100 === 0) {
|
|
if (count % 100 === 0) {
|
|
|
await new Promise<void>(r => setImmediate(r));
|
|
await new Promise<void>(r => setImmediate(r));
|
|
|
}
|
|
}
|
|
|
|
|
+ } else {
|
|
|
|
|
+ tallySkip(stats, filePath);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
return files;
|
|
return files;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- return scanDirectoryWalk(rootDir, onProgress);
|
|
|
|
|
|
|
+ return scanDirectoryWalk(rootDir, onProgress, stats);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -1441,7 +1474,8 @@ export async function scanDirectoryAsync(
|
|
|
*/
|
|
*/
|
|
|
function scanDirectoryWalk(
|
|
function scanDirectoryWalk(
|
|
|
rootDir: string,
|
|
rootDir: string,
|
|
|
- onProgress?: (current: number, file: string) => void
|
|
|
|
|
|
|
+ onProgress?: (current: number, file: string) => void,
|
|
|
|
|
+ stats?: ScanSkipStats
|
|
|
): string[] {
|
|
): string[] {
|
|
|
const files: string[] = [];
|
|
const files: string[] = [];
|
|
|
let count = 0;
|
|
let count = 0;
|
|
@@ -1524,10 +1558,14 @@ function scanDirectoryWalk(
|
|
|
walk(fullPath, active);
|
|
walk(fullPath, active);
|
|
|
}
|
|
}
|
|
|
} else if (stat.isFile()) {
|
|
} else if (stat.isFile()) {
|
|
|
- if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
|
|
|
|
|
- files.push(relativePath);
|
|
|
|
|
- count++;
|
|
|
|
|
- onProgress?.(count, relativePath);
|
|
|
|
|
|
|
+ if (!isIgnored(fullPath, false, active)) {
|
|
|
|
|
+ if (isSourceFile(relativePath, overrides)) {
|
|
|
|
|
+ files.push(relativePath);
|
|
|
|
|
+ count++;
|
|
|
|
|
+ onProgress?.(count, relativePath);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ tallySkip(stats, relativePath);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
} catch {
|
|
} catch {
|
|
@@ -1541,10 +1579,14 @@ function scanDirectoryWalk(
|
|
|
walk(fullPath, active);
|
|
walk(fullPath, active);
|
|
|
}
|
|
}
|
|
|
} else if (entry.isFile()) {
|
|
} else if (entry.isFile()) {
|
|
|
- if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
|
|
|
|
|
- files.push(relativePath);
|
|
|
|
|
- count++;
|
|
|
|
|
- onProgress?.(count, relativePath);
|
|
|
|
|
|
|
+ if (!isIgnored(fullPath, false, active)) {
|
|
|
|
|
+ if (isSourceFile(relativePath, overrides)) {
|
|
|
|
|
+ files.push(relativePath);
|
|
|
|
|
+ count++;
|
|
|
|
|
+ onProgress?.(count, relativePath);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ tallySkip(stats, relativePath);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -1791,6 +1833,7 @@ export class ExtractionOrchestrator {
|
|
|
// early-run 5-10s single stalls were observed on 95k-file repos but never
|
|
// early-run 5-10s single stalls were observed on 95k-file repos but never
|
|
|
// attributed — these labels settle scan vs framework-detect vs grammars.
|
|
// attributed — these labels settle scan vs framework-detect vs grammars.
|
|
|
const tScan = Date.now();
|
|
const tScan = Date.now();
|
|
|
|
|
+ const skipStats: ScanSkipStats = { unsupportedByExtension: new Map() };
|
|
|
const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
|
|
const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
|
|
|
onProgress?.({
|
|
onProgress?.({
|
|
|
phase: 'scanning',
|
|
phase: 'scanning',
|
|
@@ -1798,8 +1841,20 @@ export class ExtractionOrchestrator {
|
|
|
total: 0,
|
|
total: 0,
|
|
|
currentFile: file,
|
|
currentFile: file,
|
|
|
});
|
|
});
|
|
|
- });
|
|
|
|
|
|
|
+ }, skipStats);
|
|
|
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
|
|
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
|
|
|
|
|
+ /** Only meaningful when nothing was indexable — see IndexResult (#1502). */
|
|
|
|
|
+ const skipSummary = (): Pick<IndexResult, 'filesSkippedUnsupported' | 'topUnsupportedExtensions'> => {
|
|
|
|
|
+ let total = 0;
|
|
|
|
|
+ for (const n of skipStats.unsupportedByExtension.values()) total += n;
|
|
|
|
|
+ if (total === 0) return {};
|
|
|
|
|
+ const top = [...skipStats.unsupportedByExtension.entries()]
|
|
|
|
|
+ .map(([ext, count]) => ({ ext, count }))
|
|
|
|
|
+ .sort((a, b) => b.count - a.count || a.ext.localeCompare(b.ext))
|
|
|
|
|
+ .slice(0, 5);
|
|
|
|
|
+ return { filesSkippedUnsupported: total, topUnsupportedExtensions: top };
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
|
|
|
// A re-index over an existing DB skips unchanged-hash files at the store,
|
|
// A re-index over an existing DB skips unchanged-hash files at the store,
|
|
|
// which would preserve wiped zero-node rows (#1541) — drop them first so
|
|
// which would preserve wiped zero-node rows (#1541) — drop them first so
|
|
@@ -2191,6 +2246,7 @@ export class ExtractionOrchestrator {
|
|
|
filesSkipped,
|
|
filesSkipped,
|
|
|
filesErrored,
|
|
filesErrored,
|
|
|
filesDiscovered: total,
|
|
filesDiscovered: total,
|
|
|
|
|
+ ...skipSummary(),
|
|
|
nodesCreated: totalNodes,
|
|
nodesCreated: totalNodes,
|
|
|
edgesCreated: totalEdges,
|
|
edgesCreated: totalEdges,
|
|
|
errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
|
|
errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
|
|
@@ -2347,6 +2403,7 @@ export class ExtractionOrchestrator {
|
|
|
filesSkipped,
|
|
filesSkipped,
|
|
|
filesErrored,
|
|
filesErrored,
|
|
|
filesDiscovered: total,
|
|
filesDiscovered: total,
|
|
|
|
|
+ ...skipSummary(),
|
|
|
nodesCreated: totalNodes,
|
|
nodesCreated: totalNodes,
|
|
|
edgesCreated: totalEdges,
|
|
edgesCreated: totalEdges,
|
|
|
errors,
|
|
errors,
|