From 238dbc5cec0ff65fe79e6893de7fe22288b7ff61 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Thu, 20 Aug 2026 09:10:07 -0700 Subject: [PATCH 01/27] fix(explore): accurately resolve query file paths and find camelCase symbols Previously, `codegraph_explore` queries explicitly naming files by path (e.g., `src/routes/m/projects/[id]/runs/[runId]/+page.svelte`) were shredded. Bracketed path segments exploded into "named symbol" seeds, and FTS on fragments like `page` or `runs` admitted every sibling file, starving the user's intended target. This change introduces: - **Query path pinning:** File paths named in a query are now resolved against the index, "pinned," and stripped from the query. Pinned files are guaranteed inclusion, top ranking, and fair allocation. Unresolvable path-like spans are reported. - **Segment vocabulary supplement:** Natural language query terms (e.g., "auto-scroll to bottom") can now reach camelCase identifiers (e.g., `pinFeedIfNearBottom`, `feedAtBottom`) by matching against their constituent segments. - **Variable seeding:** `variable` and `constant` node kinds are now included in identifier seeding, improving recall for `$state`-style variables common in frameworks like Svelte. --- __tests__/explore-path-pinning.test.ts | 105 +++++++++ __tests__/explore-pinned-allocation.test.ts | 83 +++++++ .../explore-path-pinning/package.json | 5 + .../src/lib/runs-store.ts | 31 +++ .../routes/(protected)/chat-window/+page.ts | 28 +++ .../m/projects/[id]/runs/[runId]/+page.ts | 67 ++++++ __tests__/identifier-segments.test.ts | 30 +++ __tests__/query-paths.test.ts | 129 +++++++++++ src/context/index.ts | 37 ++- src/index.ts | 20 +- src/mcp/explore-diagnostics.ts | 4 + src/mcp/tools.ts | 148 ++++++++++-- src/search/identifier-segments.ts | 27 +++ src/search/query-paths.ts | 213 ++++++++++++++++++ src/types.ts | 9 + 15 files changed, 905 insertions(+), 31 deletions(-) create mode 100644 __tests__/explore-path-pinning.test.ts create mode 100644 __tests__/explore-pinned-allocation.test.ts create mode 100644 __tests__/fixtures/explore-path-pinning/package.json create mode 100644 __tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts create mode 100644 __tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts create mode 100644 __tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts create mode 100644 __tests__/query-paths.test.ts create mode 100644 src/search/query-paths.ts diff --git a/__tests__/explore-path-pinning.test.ts b/__tests__/explore-path-pinning.test.ts new file mode 100644 index 000000000..8ec15e1f2 --- /dev/null +++ b/__tests__/explore-path-pinning.test.ts @@ -0,0 +1,105 @@ +/** + * End-to-end gate for query-path pinning + the segment-vocab supplement + + * variable seeding, on the bug that motivated all three: an agent named a + * SvelteKit route file by exact path plus behavior words ("scrollToBottom, + * onscroll, atBottom tracking") and got back neither the file's scroll code + * nor the file itself at full weight — the bracketed path was tokenizer + * shrapnel (`runId` seeded as a named symbol, every sibling `+page` admitted) + * and the camelCase scroll symbols were FTS-opaque. + * + * The fixture mirrors that shape in plain TS (bracket/paren directories are + * the crux, not the language): a target file under + * `src/routes/m/projects/[id]/runs/[runId]/` holding `feedAtBottom` / + * `handleFeedScroll` / `pinFeedIfNearBottom`, a decoy chat-window page under + * a `(protected)` route group, and a runs-store decoy defining `runId` and + * `Scope` — the two symbols that headlined the original junk blast radius. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +const FIXTURE = 'explore-path-pinning'; +const TARGET = 'src/routes/m/projects/[id]/runs/[runId]/+page.ts'; +const DECOY_CHAT = 'src/routes/(protected)/chat-window/+page.ts'; + +let dir: string; +let cg: CodeGraph; + +async function explore(query: string): Promise { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return res.content?.[0]?.text ?? ''; +} + +/** The response renders a source section for `file`. */ +const hasSection = (response: string, file: string): boolean => + response.includes('**`' + file + '`'); + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-path-pin-')); + fs.cpSync(path.join(__dirname, 'fixtures', FIXTURE), dir, { recursive: true }); + fs.rmSync(path.join(dir, '.codegraph'), { recursive: true, force: true }); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('fixture shape — if this rots, the gates below mean nothing', () => { + it('indexes the bracketed-path target with its scroll symbols', () => { + const names = cg.getNodesInFile(TARGET).map((n) => n.name); + expect(names).toContain('feedAtBottom'); + expect(names).toContain('handleFeedScroll'); + expect(names).toContain('pinFeedIfNearBottom'); + }); +}); + +describe('path pinning (fix 1)', () => { + it('a pure-path query renders the named file and says it was pinned', async () => { + const out = await explore(TARGET); + expect(hasSection(out, TARGET)).toBe(true); + expect(out).toContain('pinned from the query'); + }); + + it('the original bug-shaped query renders the pinned file, not path shrapnel', async () => { + const out = await explore( + `run page auto-scroll to bottom logic in ${TARGET} — scrollToBottom, onscroll, atBottom tracking`, + ); + expect(hasSection(out, TARGET)).toBe(true); + // The path fragments must not seed: `runId` (runs-store decoy) and the + // bracketed segment's namesakes headlined the original junk blast radius. + const blast = out.split('**Relationships**')[0]!; + expect(blast).not.toMatch(/`runId` \(src\/lib\/runs-store\.ts/); + // The chat decoy MAY render — it genuinely holds scroll-pinning code the + // segment supplement now finds — but the pinned file must rank first. + // (Pre-fix, `+page`/`runs` shrapnel admitted the siblings ABOVE the named + // file and the envelope truncated it.) + const decoyAt = out.indexOf('**`' + DECOY_CHAT + '`'); + const targetAt = out.indexOf('**`' + TARGET + '`'); + expect(targetAt).toBeGreaterThan(-1); + if (decoyAt !== -1) expect(targetAt).toBeLessThan(decoyAt); + }); + + it('an unresolvable path is reported, not silently dropped', async () => { + const out = await explore('crash in src/routes/gone/missing-page.ts on load'); + expect(out).toContain('No indexed file uniquely matches'); + expect(out).toContain('src/routes/gone/missing-page.ts'); + }); +}); + +describe('segment supplement + variable seeding (fixes 2–3)', () => { + it('word-level scroll terms reach the camelCase scroll code without a path', async () => { + const out = await explore('feed auto-scroll to bottom pinning behavior'); + expect(hasSection(out, TARGET)).toBe(true); + }); + + it('a camel infix naming only $state-style variables still finds their file', async () => { + const out = await explore('where does the atBottom flag get reset'); + expect(hasSection(out, TARGET)).toBe(true); + }); +}); diff --git a/__tests__/explore-pinned-allocation.test.ts b/__tests__/explore-pinned-allocation.test.ts new file mode 100644 index 000000000..e093c8ad2 --- /dev/null +++ b/__tests__/explore-pinned-allocation.test.ts @@ -0,0 +1,83 @@ +/** + * Pinned files in `allocateExploreBudget` (see query-paths.ts): a file the + * query named by PATH must survive every allocation guard. Its score is + * whatever the path-stripped query happened to match — for a pure-path query, + * nearly nothing — so without the pinned floor the proportional split would + * fund the one file the agent explicitly asked for worst of all, and the + * cliff would zero it outright. + */ +import { describe, it, expect } from 'vitest'; +import { allocateExploreBudget, getExploreOutputBudget, EXPLORE_ALLOCATION } from '../src/mcp/tools'; +import type { ExploreAllocationCandidate } from '../src/mcp/tools'; + +const cand = ( + path: string, + score: number, + extra: Partial = {}, +): ExploreAllocationCandidate => ({ path, score, worth: 1, spine: false, ...extra }); + +const budget = getExploreOutputBudget(1000); + +describe('allocateExploreBudget — pinned files', () => { + it('never cliffs a pinned file, however low it scores', () => { + const { allowances, cliffed } = allocateExploreBudget( + [ + cand('pinned.svelte', 0.1, { pinned: true }), + cand('hub.ts', 200), + cand('noise.ts', 0.1), + ], + budget, + 8, + ); + expect(cliffed).toContain('noise.ts'); + expect(cliffed).not.toContain('pinned.svelte'); + expect(allowances.has('pinned.svelte')).toBe(true); + }); + + it('funds a pinned file at least as well as the strongest candidate', () => { + const { allowances } = allocateExploreBudget( + [ + cand('pinned.svelte', 0.5, { pinned: true }), + cand('hub.ts', 300), + cand('helper.ts', 40), + ], + budget, + 8, + ); + expect(allowances.get('pinned.svelte')!).toBeGreaterThanOrEqual(allowances.get('hub.ts')!); + expect(allowances.get('pinned.svelte')!).toBeGreaterThan(allowances.get('helper.ts')!); + }); + + it('keeps pinned files through the affordability trim', () => { + // Smallest tier: affordable = floor(13000 / (MIN_CHARS + FILE_OVERHEAD)) = 14 + // slots. 18 equal-weight candidates admitted → the trim must cut 4. The + // pinned file sits last with a TIED weight (the pinned floor lifts it to + // the top weight), so the stable by-weight sort would slice it off — only + // the explicit spine/pinned keep saves it. + const tiny = getExploreOutputBudget(10); + const fleet = Array.from({ length: 17 }, (_, i) => cand(`f${i}.ts`, 100)); + fleet.push(cand('pinned.svelte', 0.1, { pinned: true })); + const { allowances, cliffed } = allocateExploreBudget(fleet, tiny, 18); + expect(allowances.has('pinned.svelte')).toBe(true); + expect(cliffed).not.toContain('pinned.svelte'); + expect(allowances.size).toBeLessThan(18); + }); + + it('an all-pinned zero-score call still allocates (pure-path query)', () => { + const { allowances, pool } = allocateExploreBudget( + [cand('a.svelte', 0, { pinned: true }), cand('b.svelte', 0, { pinned: true })], + budget, + 8, + ); + expect(pool).toBeGreaterThan(0); + expect(allowances.get('a.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + expect(allowances.get('b.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + }); + + it('unpinned behavior is unchanged when no candidate is pinned', () => { + const before = allocateExploreBudget( + [cand('a.ts', 40), cand('b.ts', 10)], budget, 8, + ); + expect(before.allowances.get('a.ts')!).toBeGreaterThan(before.allowances.get('b.ts')!); + }); +}); diff --git a/__tests__/fixtures/explore-path-pinning/package.json b/__tests__/fixtures/explore-path-pinning/package.json new file mode 100644 index 000000000..c17caa943 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/package.json @@ -0,0 +1,5 @@ +{ + "name": "explore-path-pinning-fixture", + "version": "1.0.0", + "private": true +} diff --git a/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts b/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts new file mode 100644 index 000000000..b3a39d889 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts @@ -0,0 +1,31 @@ +/** In-memory registry of task runs, keyed by run id. */ + +export interface Scope { + projectId: string; + label: string; +} + +export const runId = 'run-000'; + +const runs = new Map(); + +export function registerRun(id: string, scope: Scope): void { + runs.set(id, { id, scope, status: 'queued' }); +} + +export function getRun(id: string): { id: string; scope: Scope; status: string } | null { + return runs.get(id) ?? null; +} + +export function listRuns(scope: Scope): string[] { + return [...runs.values()] + .filter((r) => r.scope.projectId === scope.projectId) + .map((r) => r.id); +} + +export function stopRun(id: string): boolean { + const run = runs.get(id); + if (!run) return false; + run.status = 'cancelled'; + return true; +} diff --git a/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts b/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts new file mode 100644 index 000000000..3023ba459 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts @@ -0,0 +1,28 @@ +/** Detached chat window page — session presence + streaming state. */ + +let chatAtBottom = true; +let isStreaming = false; +let messages: string[] = []; + +export function handleMessagesScroll(distance: number): void { + chatAtBottom = distance < 50; +} + +export function sendMessage(text: string): void { + messages = [...messages, text]; + isStreaming = true; +} + +export function stopResponse(): void { + isStreaming = false; +} + +export function redock(): void { + messages = []; + isStreaming = false; + chatAtBottom = true; +} + +export function chatSnapshot(): { messages: string[]; streaming: boolean } { + return { messages: [...messages], streaming: isStreaming }; +} diff --git a/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts b/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts new file mode 100644 index 000000000..c5fe2b77c --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts @@ -0,0 +1,67 @@ +/** Mobile run feed — event stream + scroll pinning for the run page. */ + +export interface FeedEvent { + id: string; + kind: 'output' | 'tool' | 'error'; + content: string; +} + +const EVENT_CAP = 300; + +let events: FeedEvent[] = []; +let workingLine: string | null = null; + +/** Whether the reader is at the tail of the feed (within 50px). */ +let feedAtBottom = true; + +interface FeedElement { + scrollTop: number; + scrollHeight: number; + clientHeight: number; +} + +let feedEl: FeedElement | null = null; + +export function bindFeedElement(el: FeedElement | null): void { + feedEl = el; +} + +/** Track the reader's position; called from the feed's scroll listener. */ +export function handleFeedScroll(): void { + const el = feedEl; + if (!el) return; + feedAtBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 50; +} + +/** Re-pin after async content growth (image loads), only when at the tail. */ +export function pinFeedIfNearBottom(): void { + const el = feedEl; + if (!el) return; + if (feedAtBottom) { + el.scrollTop = el.scrollHeight; + } +} + +export function appendEvent(event: FeedEvent): void { + events = events.length >= EVENT_CAP + ? [...events.slice(-(EVENT_CAP - 1)), event] + : [...events, event]; + if (feedAtBottom) { + pinFeedIfNearBottom(); + } +} + +export function setWorkingLine(line: string | null): void { + workingLine = line; + pinFeedIfNearBottom(); +} + +export function resetFeed(): void { + events = []; + workingLine = null; + feedAtBottom = true; +} + +export function feedSnapshot(): { events: FeedEvent[]; workingLine: string | null } { + return { events: [...events], workingLine }; +} diff --git a/__tests__/identifier-segments.test.ts b/__tests__/identifier-segments.test.ts index 11884bf0a..477c702a1 100644 --- a/__tests__/identifier-segments.test.ts +++ b/__tests__/identifier-segments.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest'; import { splitIdentifierSegments, extractProseCandidates, + extractSegmentSearchWords, normalizeProseWord, segmentLookupVariants, } from '../src/search/identifier-segments'; @@ -101,3 +102,32 @@ describe('segmentLookupVariants — light plural folding', () => { expect(segmentLookupVariants('boxes')).toEqual(['boxes']); // -es strip would go sub-minimum }); }); + +describe('extractSegmentSearchWords — query words for the search-side vocab supplement', () => { + it('keeps prose words and adds camel-token segments', () => { + const words = extractSegmentSearchWords('auto-scroll to bottom — atBottom tracking'); + // Prose candidates survive as before… + expect(words).toContain('scroll'); + expect(words).toContain('bottom'); + expect(words).toContain('tracking'); + // …and the camel token contributed its ≥4-char segments ("at" is under + // the prose minimum; "bottom" arrives from the split even when the prose + // pass missed it). + expect(extractSegmentSearchWords('where is atBottom set')).toContain('bottom'); + }); + + it('splits multi-hump tokens into every usable segment', () => { + const words = extractSegmentSearchWords('trace pinFeedIfNearBottom please'); + expect(words).toEqual(expect.arrayContaining(['feed', 'near', 'bottom'])); + }); + + it('does not invent segments for plain prose', () => { + const words = extractSegmentSearchWords('how does checkout work'); + expect(words).toContain('checkout'); + expect(words).not.toContain('check'); + }); + + it('returns nothing for an empty query', () => { + expect(extractSegmentSearchWords('')).toEqual([]); + }); +}); diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts new file mode 100644 index 000000000..13a47470e --- /dev/null +++ b/__tests__/query-paths.test.ts @@ -0,0 +1,129 @@ +/** + * File-path recognition in explore queries (src/search/query-paths.ts). + * + * The originating bug: an agent named two SvelteKit route files by exact path + * (`src/routes/m/projects/[id]/runs/[runId]/+page.svelte`) and the explore + * pipeline shredded them — the seeding tokenizer splits on brackets, so the + * fragments `runId`/`scope` seeded as "named symbols" and headlined the blast + * radius, while FTS admitted every sibling `+page.svelte` off the `page`/`runs` + * fragments. These tests pin the module that stops that: path spans resolve + * against the indexed file list, matching files pin, and the spans leave the + * query. Resolution IS the detector — slash-bearing non-paths stay untouched. + */ +import { describe, it, expect } from 'vitest'; +import { extractQueryPaths, queryMightContainPaths } from '../src/search/query-paths'; + +const INDEX = [ + 'src/routes/m/projects/[id]/runs/[runId]/+page.svelte', + 'src/routes/m/projects/[id]/chat/[scope]/+page.svelte', + 'src/routes/m/projects/[id]/+page.svelte', + 'src/routes/(protected)/chat-window/+page.svelte', + 'src/lib/chat-manager.ts', + 'src/lib/task-runner-manager.ts', + 'src/lib/stores/sqlite-store.ts', + 'src/lib/stores/postgresql-store.ts', +]; + +describe('queryMightContainPaths — the cheap pre-gate', () => { + it('fires on slashes and dotted basenames', () => { + expect(queryMightContainPaths('look at src/lib/chat-manager.ts')).toBe(true); + expect(queryMightContainPaths('look at chat-manager.ts please')).toBe(true); + }); + + it('stays quiet on plain prose and Class.method spans', () => { + expect(queryMightContainPaths('how does the scroll pinning work')).toBe(false); + // `.isPackaged` is 10 chars — past the 8-char extension cap. + expect(queryMightContainPaths('what reads app.isPackaged here')).toBe(false); + }); +}); + +describe('extractQueryPaths — resolution and stripping', () => { + it('resolves a bracketed SvelteKit path and strips it from the query', () => { + const q = 'auto-scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte — atBottom tracking'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual(['src/routes/m/projects/[id]/runs/[runId]/+page.svelte']); + expect(out.strippedQuery).not.toContain('+page.svelte'); + expect(out.strippedQuery).not.toContain('runId'); + expect(out.strippedQuery).toContain('atBottom tracking'); + expect(out.unresolvedPathSpans).toEqual([]); + }); + + it('pins multiple named files in appearance order', () => { + const q = 'compare src/routes/m/projects/[id]/chat/[scope]/+page.svelte and src/routes/m/projects/[id]/runs/[runId]/+page.svelte'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual([ + 'src/routes/m/projects/[id]/chat/[scope]/+page.svelte', + 'src/routes/m/projects/[id]/runs/[runId]/+page.svelte', + ]); + }); + + it('resolves a (protected) route-group path — parens are path characters', () => { + const out = extractQueryPaths('read src/routes/(protected)/chat-window/+page.svelte', INDEX); + expect(out.pinnedFiles).toEqual(['src/routes/(protected)/chat-window/+page.svelte']); + }); + + it('resolves an absolute path by walking suffixes to the indexed relative path', () => { + const q = 'fix /Users/colby/dev/beads-live-dashboard/src/lib/chat-manager.ts'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('resolves a unique basename and a partial path', () => { + expect(extractQueryPaths('see chat-manager.ts', INDEX).pinnedFiles) + .toEqual(['src/lib/chat-manager.ts']); + expect(extractQueryPaths('see stores/sqlite-store.ts', INDEX).pinnedFiles) + .toEqual(['src/lib/stores/sqlite-store.ts']); + }); + + it('strips wrapping punctuation and line references', () => { + const out = extractQueryPaths('the bug (see `src/lib/chat-manager.ts:243`).', INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + const hash = extractQueryPaths('regression at src/lib/task-runner-manager.ts#L88-L120', INDEX); + expect(hash.pinnedFiles).toEqual(['src/lib/task-runner-manager.ts']); + }); + + it('treats an over-ambiguous basename as unresolved — stripped and reported', () => { + const out = extractQueryPaths('why do all +page.svelte files flash', INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual(['+page.svelte']); + expect(out.strippedQuery).toBe('why do all files flash'); + }); + + it('strips and reports a clearly-path-shaped span that matches nothing', () => { + const out = extractQueryPaths('crash in src/routes/gone/missing-page.svelte on load', INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual(['src/routes/gone/missing-page.svelte']); + expect(out.strippedQuery).toBe('crash in on load'); + }); + + it('leaves slash-bearing non-paths alone', () => { + const q = 'does gen_server:call/2 block and/or timeout'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual([]); + expect(out.strippedQuery).toBe(q); + }); + + it('dedupes a path named twice and honors maxPins', () => { + const twice = extractQueryPaths( + 'src/lib/chat-manager.ts wraps src/lib/chat-manager.ts', INDEX, + ); + expect(twice.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + + const capped = extractQueryPaths( + 'src/lib/chat-manager.ts src/lib/task-runner-manager.ts', INDEX, { maxPins: 1 }, + ); + expect(capped.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('matches case-insensitively but returns the indexed spelling', () => { + const out = extractQueryPaths('SRC/LIB/CHAT-MANAGER.TS', INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('passes through untouched when nothing resolves', () => { + const q = 'plain prose question about scrolling'; + const out = extractQueryPaths(q, INDEX); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + }); +}); diff --git a/src/context/index.ts b/src/context/index.ts index ad4d63bc0..a3da12c40 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -171,6 +171,7 @@ const DEFAULT_FIND_OPTIONS: Required = { minScore: 0.3, edgeKinds: [], nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default + seedNames: [], // Segment-vocab supplement — filled by the facade }; // Re-export the low-confidence sentinel (defined in a dependency-free leaf so @@ -460,13 +461,37 @@ export class ContextBuilder { // Step 2: Look up exact matches for extracted symbols let exactMatches: SearchResult[] = []; - if (symbolsFromQuery.length > 0) { + if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0) { try { - // Get more results so we can apply co-location boosting before trimming - exactMatches = this.queries.findNodesByExactName(symbolsFromQuery, { - limit: Math.ceil(opts.searchLimit * 5), - kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, - }); + if (symbolsFromQuery.length > 0) { + // Get more results so we can apply co-location boosting before trimming + exactMatches = this.queries.findNodesByExactName(symbolsFromQuery, { + limit: Math.ceil(opts.searchLimit * 5), + kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + }); + } + + // Step 2a: segment-vocabulary seeds. Word-level query terms cannot + // reach camelCase names through FTS (one token per name), so the + // caller resolves query words → names via the segment vocab and hands + // them in as seedNames. Merged at a dampened score — a symbol the + // query names outright must outrank a segment-derived one — but + // BEFORE the co-location boost below, because several seeds landing + // in one file (pinFeedIfNearBottom + feedAtBottom + handleFeedScroll) + // is exactly the evidence that file is the answer. + if (opts.seedNames.length > 0) { + const seedResults = this.queries.findNodesByExactName(opts.seedNames, { + limit: Math.ceil(opts.searchLimit * 3), + kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + }); + const known = new Set(exactMatches.map((r) => r.node.id)); + for (const r of seedResults) { + if (known.has(r.node.id)) continue; + known.add(r.node.id); + exactMatches.push({ ...r, score: r.score * 0.6 }); + } + logDebug('Segment seed matches', { seedNames: opts.seedNames, added: known.size }); + } // Co-location boost: when multiple extracted symbols appear in the same file, // those results are much more likely to be what the user is looking for. diff --git a/src/index.ts b/src/index.ts index a05661c82..15eaf7a7a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -54,7 +54,7 @@ import { EXTRACTION_VERSION } from './extraction/extraction-version'; import { getCodeGraphDir } from './directory'; import { deriveProjectNameTokens } from './search/query-utils'; import { CodeGraphPackageVersion } from './mcp/version'; -import { segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; +import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; import { createYielder } from './resolution/cooperative-yield'; import { minRefsForPool } from './resolution/resolver-pool'; @@ -1823,7 +1823,23 @@ export class CodeGraph { query: string, options?: FindRelevantContextOptions ): Promise { - return this.contextBuilder.findRelevantContext(query, options); + // Segment-vocab supplement: FTS keeps camelCase names as single tokens, + // so a word-level query ("auto-scroll to bottom") can never reach + // `pinFeedIfNearBottom` through search alone. Resolve the query's words + // against name_segment_vocab (same precision rules as the prompt hook: + // co-occurrence, else rare singles, verified against live nodes) and hand + // the names down as dampened exact-name seeds. Callers that pass their + // own seedNames keep them; failures degrade to no supplement. + let seedNames = options?.seedNames; + if (seedNames === undefined) { + try { + seedNames = this.getSegmentMatches(extractSegmentSearchWords(query), 8) + .map((m) => m.name); + } catch { + seedNames = []; + } + } + return this.contextBuilder.findRelevantContext(query, { ...options, seedNames }); } /** diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts index 4de9c605c..0c654e025 100644 --- a/src/mcp/explore-diagnostics.ts +++ b/src/mcp/explore-diagnostics.ts @@ -60,6 +60,8 @@ export interface ExploreCandidateMeta { graphScore: number; termHits: number; nodes: number; + /** The query named this file by PATH — pinned rank/allocation treatment. */ + pinned?: boolean; named: boolean; central: boolean; entry: boolean; @@ -579,6 +581,7 @@ export class ExploreDiagnostics { graphScore: round6(r.graphScore), termHits: r.termHits, nodes: r.nodes, + pinned: r.pinned ?? false, named: r.named, central: r.central, entry: r.entry, @@ -797,6 +800,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { function flagString(f: ExploreDiagnosticFile): string { const flags: string[] = []; + if (f.pinned) flags.push('pinned'); if (f.named) flags.push('named'); if (f.entry) flags.push('entry'); if (f.central) flags.push('central'); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index d1d013514..4ad7e64b6 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -32,6 +32,7 @@ import { import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; import { isTestFile, normalizeNameToken } from '../search/query-utils'; +import { extractQueryPaths, queryMightContainPaths } from '../search/query-paths'; import { existsSync, readFileSync, @@ -631,6 +632,13 @@ export interface ExploreAllocationCandidate { worth: number; /** Carries a symbol on the rendered flow spine. */ spine: boolean; + /** + * The query named this file by PATH (see query-paths.ts). Pinned files are + * never cliffed or trimmed, and weigh at least as much as the strongest + * candidate — the agent asked for the file itself, so starving it on text/ + * graph scores (which a pure-path query doesn't produce) defeats the ask. + */ + pinned?: boolean; } export interface ExploreAllocation { @@ -676,7 +684,16 @@ export function allocateExploreBudget( return Number.isFinite(w) ? w : 0; }; - const weights = new Map(candidates.map((c) => [c.path, weightOf(c)])); + // Pinned files weigh at least as much as the strongest raw candidate: their + // score is whatever the stripped query happened to match (for a pure-path + // query, nearly nothing), and a proportional split on that would fund the + // named file worst of all. Floor of 1 covers the all-pinned/zero-score case. + const rawWeights = new Map(candidates.map((c) => [c.path, weightOf(c)])); + const topRaw = Math.max(...rawWeights.values()); + const weights = new Map(candidates.map((c) => [ + c.path, + c.pinned ? Math.max(rawWeights.get(c.path) ?? 0, topRaw, 1) : (rawWeights.get(c.path) ?? 0), + ])); const topWeight = Math.max(...weights.values()); if (!(topWeight > 0)) return empty; @@ -686,7 +703,7 @@ export function allocateExploreBudget( const cliffed: string[] = []; let admitted: ExploreAllocationCandidate[] = []; for (const c of candidates) { - if (!c.spine && (weights.get(c.path) ?? 0) < cliffAt) cliffed.push(c.path); + if (!c.spine && !c.pinned && (weights.get(c.path) ?? 0) < cliffAt) cliffed.push(c.path); else admitted.push(c); } // Never cliff every candidate: an empty response costs a whole round-trip. @@ -705,7 +722,7 @@ export function allocateExploreBudget( if (admitted.length > affordable) { const byWeight = [...admitted].sort((a, b) => (weights.get(b.path) ?? 0) - (weights.get(a.path) ?? 0)); const keep = new Set(byWeight.slice(0, affordable).map((c) => c.path)); - for (const c of admitted) if (c.spine) keep.add(c.path); + for (const c of admitted) if (c.spine || c.pinned) keep.add(c.path); for (const c of admitted) if (!keep.has(c.path)) cliffed.push(c.path); admitted = admitted.filter((c) => keep.has(c.path)); } @@ -3223,6 +3240,34 @@ export class ToolHandler { } const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + // File paths named in the query become PINNED files: guaranteed admission, + // top of the rank order, funded first — and their span is REMOVED from the + // matching query. Runs on the RAW query (normalizeQuerySpelling strips + // `/digits` tails, which would mangle numeric path segments). Without this, + // a SvelteKit path like `runs/[runId]/+page.svelte` was shredded by the + // seeding tokenizer (splits on brackets → `runId` seeded as a "named + // symbol") and by FTS (`page`/`runs` fragments admitted every sibling + // `+page.svelte`), starving the very files the agent asked for. + let pinnedFiles: string[] = []; + let unresolvedPathSpans: string[] = []; + let matchQuery = query; + if (queryMightContainPaths(rawQuery)) { + try { + const extraction = extractQueryPaths( + rawQuery, + cg.getFiles().map((f) => f.path), + { maxPins: maxFiles }, + ); + if (extraction.pinnedFiles.length > 0 || extraction.unresolvedPathSpans.length > 0) { + pinnedFiles = extraction.pinnedFiles; + unresolvedPathSpans = extraction.unresolvedPathSpans; + matchQuery = normalizeQuerySpelling(extraction.strippedQuery); + } + } catch { /* path pinning must never fail an explore call */ } + } + const pinnedSet = new Set(pinnedFiles); + const pinnedOrder = new Map(pinnedFiles.map((p, i) => [p, i])); + // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG // is set — every `diag?.` below is then a no-op and the response is // byte-identical. It only OBSERVES: it must never feed back into rendering. @@ -3279,16 +3324,34 @@ export class ToolHandler { // Use a large maxNodes budget — explore has its own 35k char output limit // that prevents context bloat, so more nodes just means better coverage // across entry points (especially for large files like Svelte components). - const subgraph = await cg.findRelevantContext(query, { + // Matching runs on the path-stripped query; `query` stays for display. + const subgraph = await cg.findRelevantContext(matchQuery, { searchLimit: 8, traversalDepth: 3, maxNodes: 200, minScore: 0.2, }); + // Pinned files' symbols enter the gather unconditionally — the agent named + // the file itself, so its contents ARE the answer regardless of what the + // stripped query text matched (which, for a pure-path query, is nothing). + const PINNED_FILE_NODE_CAP = 300; + for (const fp of pinnedFiles) { + let fileNodes: Node[] = []; + try { fileNodes = cg.getNodesInFile(fp); } catch { continue; } + fileNodes + .filter((n) => n.kind !== 'file' && n.kind !== 'import' && n.kind !== 'export') + .sort((a, b) => a.startLine - b.startLine) + .slice(0, PINNED_FILE_NODE_CAP) + .forEach((n) => { if (!subgraph.nodes.has(n.id)) subgraph.nodes.set(n.id, n); }); + } + if (subgraph.nodes.size === 0) { diag?.finishEmpty('no relevant code found — empty subgraph'); - const empty = `No relevant code found for "${query}"`; + const missNote = unresolvedPathSpans.length > 0 + ? ` (no indexed file uniquely matches ${unresolvedPathSpans.map((s) => `\`${s}\``).join(', ')})` + : ''; + const empty = `No relevant code found for "${query}"${missNote}`; // Still an explore call, so it is still recorded: an empty answer spends a // call against the tier budget even though it emits no source. return this.exploreResult(empty, { @@ -3351,11 +3414,18 @@ export class ToolHandler { { const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte|astro|erl|hrl)$/i; const CALLABLE = new Set(['method', 'function', 'component', 'constructor']); + // Variables/constants seed too: in Svelte/React a `$state` variable + // (`chatAtBottom`, `feedAtBottom`) is exactly the kind of symbol an agent + // names in a query, and the exact-name search channel already returns + // them — only this seeding tier was callable-only. The NL-stopword guard + // below applies unchanged, so bare English words still can't seed a + // same-named local. Callables keep priority via the body-size sort. + const SEEDABLE = new Set([...CALLABLE, 'variable', 'constant']); const isTestPath = (p: string) => /(^|\/)(tests?|specs?|__tests__|testdata|mocks?|fixtures?)\//i.test(p) || /\.(test|spec)\.[a-z]+$/i.test(p); const bodyLines = (n: Node) => Math.max(0, (n.endLine ?? n.startLine) - n.startLine); const callerCount = (n: Node) => { try { return cg.getCallers(n.id).length; } catch { return 0; } }; const tokens = [...new Set( - query.split(/[\s,()[\]]+/) + matchQuery.split(/[\s,()[\]]+/) .map((t) => t.replace(FILE_EXT, '').trim()) .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) )].slice(0, 16); @@ -3430,24 +3500,26 @@ export class ToolHandler { } } let cands = raw - .filter((n) => CALLABLE.has(n.kind) && !isTestPath(n.filePath)) + .filter((n) => SEEDABLE.has(n.kind) && !isTestPath(n.filePath)) .sort((a, b) => (bodyLines(b) > 1 ? 1 : 0) - (bodyLines(a) > 1 ? 1 : 0) || bodyLines(b) - bodyLines(a)); // Field-name seeding fallback (#1196): a camelCase token that names NO // definition of its own is usually an object-literal key / API field // (`profileInfo`) — no node exists, so it contributed zero seeds and // the files that DEFINE it (`getProfileInfoV2` in profileController) - // never surfaced. Seed its camel-infix definers instead: callables - // whose name contains the token at a hump boundary or as a prefix. + // never surfaced. Seed its camel-infix definers instead: seedable + // symbols (callables + variables — `atBottom` must reach the `$state` + // variables `feedAtBottom`/`chatAtBottom`) whose name contains the + // token at a hump boundary or as a prefix. // Exact-empty + camel-shaped only (bare words keep the NL-stopword // guard below), shortest-first, capped so a hot infix can't flood. if (cands.length === 0 && !isQual && /[a-z][A-Z]/.test(t)) { const lcToken = t.toLowerCase(); cands = cg .getNodesByNameSubstring(t, { - kinds: ['function', 'method', 'component'], + kinds: ['function', 'method', 'component', 'variable', 'constant'], limit: 60, }) - .filter((n) => CALLABLE.has(n.kind) && !isTestPath(n.filePath)) + .filter((n) => SEEDABLE.has(n.kind) && !isTestPath(n.filePath)) .filter((n) => { const idx = n.name.toLowerCase().indexOf(lcToken); if (idx < 0) return false; @@ -3624,8 +3696,9 @@ export class ToolHandler { fileGroups.set(node.filePath, group); } - // Extract query terms for relevance checking - const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); + // Extract query terms for relevance checking (path-stripped: a pinned + // file's own path fragments must not count as "term hits" everywhere) + const queryTerms = matchQuery.toLowerCase().split(/\s+/).filter(t => t.length >= 3); // Test/spec/icon/i18n file detector — used by the pre-floor hard filter, the // rank penalty, and the comparator deprioritization. @@ -3715,9 +3788,10 @@ export class ToolHandler { // keep-minimum then pulled two test files back in as the "spread". let candidateFiles = [...fileGroups.entries()]; { - const queryMentionsTests = /\b(test|tests|testing|spec|verify|verifies)\b/i.test(query); + const queryMentionsTests = /\b(test|tests|testing|spec|verify|verifies)\b/i.test(matchQuery); if (!queryMentionsTests) { - const nonLow = candidateFiles.filter(([p]) => !isLowValue(p)); + // A pinned file is exempt: naming a test file by path IS asking for it. + const nonLow = candidateFiles.filter(([p]) => !isLowValue(p) || pinnedSet.has(p)); if (nonLow.length >= 2) { candidateFiles = nonLow; } @@ -3732,7 +3806,9 @@ export class ToolHandler { SCORE_FLOOR_ABSOLUTE, Math.min(SCORE_FLOOR_MAX, topScore * SCORE_FLOOR_FRACTION_OF_TOP), ); - let relevantFiles = candidateFiles.filter(([, group]) => group.score >= scoreFloor); + let relevantFiles = candidateFiles.filter( + ([fp, group]) => group.score >= scoreFloor || pinnedSet.has(fp), + ); if (relevantFiles.length < SCORE_FLOOR_KEEP_MIN) { // Backfill from what the RELATIVE floor cut, best first, at two strengths: // @@ -3747,8 +3823,11 @@ export class ToolHandler { // worst outcome on the board — the agent falls straight back to grep. const minEvidence = relevantFiles.length === 0 ? Number.EPSILON : SCORE_FLOOR_ABSOLUTE; relevantFiles = candidateFiles - .filter(([, group]) => group.score >= minEvidence) - .sort((a, b) => b[1].score - a[1].score || b[1].nodes.length - a[1].nodes.length) + .filter(([fp, group]) => group.score >= minEvidence || pinnedSet.has(fp)) + .sort((a, b) => + (pinnedSet.has(b[0]) ? 1 : 0) - (pinnedSet.has(a[0]) ? 1 : 0) + || b[1].score - a[1].score + || b[1].nodes.length - a[1].nodes.length) .slice(0, Math.max(SCORE_FLOOR_KEEP_MIN, relevantFiles.length)); } diag?.setScoreFloor(scoreFloor, relevantFiles.length); @@ -3852,7 +3931,8 @@ export class ToolHandler { // never prunes below 2. if (maxGraph > 0) { const gated = relevantFiles.filter(([fp]) => - (fileGraphScore.get(fp) ?? 0) >= maxGraph * 0.06 + pinnedSet.has(fp) + || (fileGraphScore.get(fp) ?? 0) >= maxGraph * 0.06 || centralFiles.has(fp) || entryFiles.has(fp) || changeSurfaceFiles.has(fp) @@ -3908,7 +3988,15 @@ export class ToolHandler { const aPath = a[0].toLowerCase(); const bPath = b[0].toLowerCase(); - // Agent-named files first (it asked for a symbol defined here by name). + // Pinned files first of all — the agent named the FILE by path, which is + // even more explicit than naming a symbol in it. Among pins, keep the + // order they appeared in the query. + const aPin = pinnedSet.has(a[0]) ? 1 : 0; + const bPin = pinnedSet.has(b[0]) ? 1 : 0; + if (aPin !== bPin) return bPin - aPin; + if (aPin && bPin) return (pinnedOrder.get(a[0]) ?? 0) - (pinnedOrder.get(b[0]) ?? 0); + + // Agent-named files next (it asked for a symbol defined here by name). const aNamed = namedSeedFiles.has(a[0]) ? 1 : 0; const bNamed = namedSeedFiles.has(b[0]) ? 1 : 0; if (aNamed !== bNamed) return bNamed - aNamed; @@ -4010,7 +4098,7 @@ export class ToolHandler { // Compute the flow spine once — used both to prepend the Flow section (below) // and to gate adaptive source sizing: files on the spine get full source, // off-spine peers skeletonize. - const flow = this.buildFlowFromNamedSymbols(cg, query); + const flow = this.buildFlowFromNamedSymbols(cg, matchQuery); // Snapshot every ranked candidate's scoring inputs, in final sort order, so // the diagnostic can show what each file's share of the envelope was BOUGHT @@ -4031,6 +4119,7 @@ export class ToolHandler { graphScore: fileGraphScore.get(fp) ?? 0, termHits: fileTermHits.get(fp) ?? 0, nodes: group.nodes.length, + pinned: pinnedSet.has(fp), named: namedSeedFiles.has(fp), central: centralFiles.has(fp), entry: entryFiles.has(fp), @@ -4051,8 +4140,11 @@ export class ToolHandler { sortedFiles.map(([fp, group]) => ({ path: fp, score: group.score, - worth: rankPenalty(fp), + // A pinned file's bytes are worth full price by definition — the agent + // asked for the file itself, generated/test or not. + worth: pinnedSet.has(fp) ? 1 : rankPenalty(fp), spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + pinned: pinnedSet.has(fp), })), budget, maxFiles, @@ -5804,9 +5896,19 @@ export class ToolHandler { g.nodes.filter((n) => n.kind !== 'import' && n.kind !== 'export').map((n) => n.id), ).size; }, 0); - const summaryLine = survivors.length > 0 + let summaryLine = survivors.length > 0 ? `Found ${shownSymbols} symbol${shownSymbols === 1 ? '' : 's'} across ${survivors.length} file${survivors.length === 1 ? '' : 's'}.` : `Found ${subgraph.nodes.size} symbol${subgraph.nodes.size === 1 ? '' : 's'} across ${fileGroups.size} file${fileGroups.size === 1 ? '' : 's'}.`; + // Path pinning is visible, not silent: say which query-named files were + // honored, and which path spans matched nothing so the agent can correct + // them instead of trusting a response that quietly ignored the path. + const pinnedShown = pinnedFiles.filter((fp) => survivors.includes(fp)).length; + if (pinnedShown > 0) { + summaryLine += ` ${pinnedShown} file${pinnedShown === 1 ? '' : 's'} pinned from the query.`; + } + if (unresolvedPathSpans.length > 0) { + summaryLine += ` No indexed file uniquely matches ${unresolvedPathSpans.map((s) => `\`${s}\``).join(', ')}.`; + } finalText = finalText.replace(SUMMARY_SENTINEL, summaryLine); // Emit the allocation diagnostic from the FINAL text, so per-file bytes and diff --git a/src/search/identifier-segments.ts b/src/search/identifier-segments.ts index 110a6b725..75bfde4e2 100644 --- a/src/search/identifier-segments.ts +++ b/src/search/identifier-segments.ts @@ -126,6 +126,33 @@ export function extractProseCandidates(prompt: string): string[] { return [...seen]; } +/** + * Words to look up in the segment vocabulary for a SEARCH query (as opposed + * to a prompt-hook gate): the query's prose candidates PLUS the segments of + * its identifier-shaped tokens. An agent's query names concepts both ways — + * "auto-scroll to bottom" (prose) and "atBottom tracking" (camel) — and the + * camel token must still reach the segment "bottom" even though the whole + * token matches no name. Same stopword/length rules as the hook path, since + * both feeds run through {@link extractProseCandidates}. + */ +export function extractSegmentSearchWords(query: string): string[] { + if (!query) return []; + const words = new Set(extractProseCandidates(query)); + const segments: string[] = []; + for (const run of query.match(/[\p{L}\p{N}]+/gu) ?? []) { + // Only camel-humped tokens contribute segments — a plain word's + // "segments" are itself (already covered above), and snake_case arrives + // as separate runs because `_` is not a letter. + if (/[\p{Ll}\p{N}]\p{Lu}/u.test(run)) { + segments.push(...splitIdentifierSegments(run)); + } + } + if (segments.length > 0) { + for (const w of extractProseCandidates(segments.join(' '))) words.add(w); + } + return [...words]; +} + /** * Lookup variants for a prose word: the word itself plus light plural folding * ("services" → service, "dependencies" → dependencie/dependency is NOT diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts new file mode 100644 index 000000000..ce186c8a2 --- /dev/null +++ b/src/search/query-paths.ts @@ -0,0 +1,213 @@ +/** + * File-path recognition for explore queries. + * + * Agents routinely name files by path in a `codegraph_explore` query — + * "the scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte" — + * and until this module existed those spans were SHREDDED by the downstream + * tokenizers instead of being read as file references: + * + * - the named-symbol seeder splits on `[\s,()[\]]+`, so SvelteKit/Next + * bracketed segments (`[id]`, `[runId]`) and route groups (`(protected)`) + * exploded the path into fragments; the identifier-shaped survivors + * (`runId`, `scope`) then seeded as "symbols the agent named" and + * headlined the blast radius; + * - FTS saw the fragments (`page`, `chat`, `runs`) and admitted every + * sibling `+page.svelte` in the repo, which ate the output envelope and + * truncated the files the agent actually asked for. + * + * `extractQueryPaths` finds path-like spans, resolves them against the + * INDEXED file list (resolution IS the detector — `and/or`, `gen_server:call/2` + * and other slash-bearing non-paths match nothing and are left alone), and + * returns the matches as pinned files plus the query with those spans removed. + * Callers treat pinned files as first-class: guaranteed admission, top rank, + * funded first. Pure string work — no DB, no fs — so it is trivially testable + * and safe inside the query-pool workers. + */ + +export interface QueryPathExtraction { + /** The query with resolved/clearly-path spans removed, whitespace-joined. */ + strippedQuery: string; + /** Indexed file paths the query named, appearance-ordered, deduped. */ + pinnedFiles: string[]; + /** + * Spans that are unambiguously path-shaped but resolved to nothing (stale + * path, unindexed file) or to too many files (bare `+page.svelte`). Stripped + * from the query — their fragments could only mint junk matches — and + * surfaced to the agent so the miss is visible instead of silent. + */ + unresolvedPathSpans: string[]; +} + +/** + * Cheap pre-gate so callers only fetch the indexed file list when the query + * could possibly contain a path: a slash, or a dot-extension-shaped tail + * (`chat-manager.ts`). Extensions cap at 8 chars, which keeps `Class.method` + * spans (`app.isPackaged`) from qualifying. + */ +export function queryMightContainPaths(query: string): boolean { + return /[/\\]/.test(query) || /\.[A-Za-z][A-Za-z0-9]{0,7}(?=[\s,;:)\]'"`]|$)/.test(query); +} + +/** + * Longest span→suffix walk tried per span. 8 covers an absolute macOS path + * (`/Users//dev//…`) over a deeply nested repo-relative file; + * deeper prefixes buy nothing. + */ +const MAX_SUFFIX_TRIES = 8; +/** Spans examined per query — a prose sentence is not 50 paths. */ +const MAX_CANDIDATE_SPANS = 8; + +/** `name.ext` shape with a plausible source extension (no slash required). */ +const DOTTED_BASENAME = /^[^\s/\\]+\.[A-Za-z][A-Za-z0-9]{0,7}$/; + +/** + * Strip prose punctuation wrapped around a token without eating punctuation + * that is PART of the path: quotes/backticks always strip; a trailing `)`/`]` + * strips only when the token has no matching opener (so `(protected)` and + * `[id]` segments survive, while "…(see src/foo.ts)" loses its parenthesis); + * a leading `(`/`[` mirrors that. Trailing sentence punctuation strips last, + * so "src/foo.ts." resolves. + */ +function stripWrapping(token: string): string { + let s = token; + for (;;) { + const first = s[0]; + if (!first) break; + if ('\'"`<'.includes(first)) { s = s.slice(1); continue; } + if (first === '(' && !s.includes(')')) { s = s.slice(1); continue; } + if (first === '[' && !s.includes(']')) { s = s.slice(1); continue; } + if (first === '{' && !s.includes('}')) { s = s.slice(1); continue; } + break; + } + for (;;) { + const last = s[s.length - 1]; + if (!last) break; + if ('\'"`>.,;!?'.includes(last)) { s = s.slice(0, -1); continue; } + if (last === ')' && !s.includes('(')) { s = s.slice(0, -1); continue; } + if (last === ']' && !s.includes('[')) { s = s.slice(0, -1); continue; } + if (last === '}' && !s.includes('{')) { s = s.slice(0, -1); continue; } + break; + } + // Line references ride along in agent-written paths: `foo.ts:123`, + // `foo.ts:12-40`, `foo.ts#L88`. The file is what gets pinned. + s = s.replace(/(?::\d+(?:-\d+)?|#L\d+(?:-L?\d+)?)$/, ''); + return s; +} + +/** Normalize a span into the repo-relative shape the files table stores. */ +function normalizeSpan(span: string): string { + return span + .replace(/\\/g, '/') + .replace(/^(?:\.\/)+/, '') + .replace(/\/{2,}/g, '/') + .replace(/\/+$/, ''); +} + +/** Path-shaped beyond doubt: ≥2 segments and a dot-extension on the last. */ +function isClearlyPathShaped(normalized: string): boolean { + const slash = normalized.lastIndexOf('/'); + if (slash <= 0) return false; + return DOTTED_BASENAME.test(normalized.slice(slash + 1)); +} + +/** + * Resolve one normalized span against the indexed paths: exact match first, + * then segment-aligned suffix matches, dropping leading segments one at a + * time (so an absolute path, or one prefixed with the repo directory name, + * still lands on the indexed repo-relative file). Suffixes only get shorter — + * and therefore only match MORE — so the walk stops at the first suffix that + * matches anything: within budget it resolves, over budget it is ambiguous. + */ +function resolveSpan( + normalizedLower: string, + lowerToOriginal: ReadonlyMap, + maxMatches: number, +): { matches: string[]; ambiguous: boolean } { + const exact = lowerToOriginal.get(normalizedLower); + if (exact) return { matches: [exact], ambiguous: false }; + + const segments = normalizedLower.split('/').filter(Boolean); + const tries = Math.min(segments.length, MAX_SUFFIX_TRIES); + for (let drop = 0; drop < tries; drop++) { + const suffix = segments.slice(drop).join('/'); + if (!suffix) break; + const withSlash = '/' + suffix; + const matches: string[] = []; + for (const [lower, original] of lowerToOriginal) { + if (lower === suffix || lower.endsWith(withSlash)) { + matches.push(original); + if (matches.length > maxMatches) return { matches: [], ambiguous: true }; + } + } + if (matches.length > 0) return { matches, ambiguous: false }; + } + return { matches: [], ambiguous: false }; +} + +export function extractQueryPaths( + query: string, + indexedPaths: readonly string[], + opts: { maxPins?: number; maxMatchesPerSpan?: number } = {}, +): QueryPathExtraction { + const maxPins = Math.max(1, opts.maxPins ?? 8); + const maxMatchesPerSpan = Math.max(1, opts.maxMatchesPerSpan ?? 3); + + const passthrough: QueryPathExtraction = { + strippedQuery: query, + pinnedFiles: [], + unresolvedPathSpans: [], + }; + if (!query.trim() || indexedPaths.length === 0) return passthrough; + + // Lowercase view of the index, built once per call. Last writer wins on a + // case-colliding pair, which is the existing file-view behavior too. + const lowerToOriginal = new Map(); + for (const p of indexedPaths) lowerToOriginal.set(p.toLowerCase(), p); + + const tokens = query.split(/\s+/).filter(Boolean); + const consumed = new Set(); + const pinned: string[] = []; + const pinnedSeen = new Set(); + const unresolved: string[] = []; + let candidatesExamined = 0; + + for (let i = 0; i < tokens.length; i++) { + if (pinned.length >= maxPins) break; + if (candidatesExamined >= MAX_CANDIDATE_SPANS) break; + const stripped = stripWrapping(tokens[i]!); + if (stripped.length < 4) continue; + const hasSlash = /[/\\]/.test(stripped); + if (!hasSlash && !DOTTED_BASENAME.test(stripped)) continue; + + const normalized = normalizeSpan(stripped); + if (!normalized) continue; + candidatesExamined++; + + const { matches, ambiguous } = resolveSpan( + normalized.toLowerCase(), lowerToOriginal, maxMatchesPerSpan, + ); + if (matches.length > 0) { + consumed.add(i); + for (const m of matches) { + if (pinnedSeen.has(m) || pinned.length >= maxPins) continue; + pinnedSeen.add(m); + pinned.push(m); + } + } else if (ambiguous || isClearlyPathShaped(normalized)) { + // A real path that didn't resolve to a usable set. Keeping it in the + // query is strictly worse — its fragments are what minted the junk + // matches this module exists to stop — so strip it and say so. + consumed.add(i); + if (unresolved.length < 4) unresolved.push(normalized); + } + // Anything else (`and/or`, `call/2`, `foo.Bar`) is not a path reference: + // leave the token for the normal matching pipeline. + } + + if (consumed.size === 0) return passthrough; + return { + strippedQuery: tokens.filter((_, i) => !consumed.has(i)).join(' '), + pinnedFiles: pinned, + unresolvedPathSpans: unresolved, + }; +} diff --git a/src/types.ts b/src/types.ts index b0ebfe433..186f57adc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -688,4 +688,13 @@ export interface FindRelevantContextOptions { /** Node types to include */ nodeKinds?: NodeKind[]; + + /** + * Extra symbol names to merge in as exact-name search candidates, at a + * dampened score. Fed by the segment-vocabulary supplement (CodeGraph. + * findRelevantContext): word-level query terms can't reach camelCase names + * through FTS — `pinFeedIfNearBottom` is one FTS token — so names whose + * SEGMENTS the query's words name are seeded here instead. + */ + seedNames?: string[]; } From d8063178979efc496280597e11c096167a6f94a2 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Thu, 20 Aug 2026 09:14:11 -0700 Subject: [PATCH 02/27] fix(explore): improve query accuracy for file paths, camelCase, and variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `codegraph_explore` previously struggled with accurately interpreting user queries. Explicitly named file paths were shredded, making it hard to target specific files; natural language queries often missed camelCase identifiers; and state held in variables was overlooked as starting symbols. This commit introduces several improvements: - **Reliable File Path Resolution:** Naming a file by its path in a `codegraph_explore` query now works reliably. The path is resolved against the index, and that file is guaranteed a place at the top of the answer. Previously, paths were broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out. A path that doesn't match any indexed file is now called out instead of silently ignored. - **CamelCase Matching for Queries:** Plainly-worded `codegraph_explore` questions now find camelCase code. A query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names. - **Variable and Constant Seeding:** Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d8ab6f79..1a0485afc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Naming a file by its path in a `codegraph_explore` query now works reliably: the path is resolved against the index and that file is guaranteed a place at the top of the answer. Previously the path was broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out of the answer entirely. A path that doesn't match any indexed file is now called out instead of silently ignored. +- Plainly-worded `codegraph_explore` questions now find camelCase code: a query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names. +- Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked. - C, C++, Objective-C and Rust unions are now indexed as first-class `union` nodes. A `union` declaration previously produced no symbol at all, so it never appeared in search or `codegraph_explore`, and anything attached to it disappeared with it — in Rust, every `impl SomeTrait for MyUnion` lost its edge, the methods from that impl were left pointing at a type the graph did not contain, and asking which types implement a trait quietly skipped the union ones. A union-shaped dispatch table in C now resolves its function pointers like a struct-shaped one. A `typedef union { … } Name;` in C keeps the typedef's name and is no longer mistaken for a plain type alias. Thanks @ctype-lab. Re-index after upgrading to pick up unions in existing projects. (#1515) - A long-lived index no longer drifts away from what a fresh `codegraph index` would produce. When a file gained or lost a symbol, references to that name in files the sync never touched kept pointing at the definition that was correct before the change, and — because nothing distinguished two same-named definitions — the winner could come down to the order files happened to be written, which differs between a full index and a sync. On this project's own repository, replaying 80 commits through `sync` left 5.7% of connections wrong; it is now 1.3%, and the wrong-answers-still-being-asserted half drops by 99.7%. Since call edges are what flow questions follow and what `codegraph_explore` ranks files by, this quietly degraded answers as an index aged, with nothing to indicate it. Syncing is unchanged in speed, and an edit that only changes a function's body does no extra work at all. Set `CODEGRAPH_NO_REBIND=1` to opt out. From 26045b3159ddaf45ea3f974dfcaefe5959430458 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Thu, 20 Aug 2026 12:16:51 -0500 Subject: [PATCH 03/27] fix(extraction): decode kernel results in indexAll retry passes; self-heal wiped rows (#1541) (#1575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parse-pool workers return kernel-language extractions as an undecoded buffer transport (nodes/edges EMPTY, tables in kernelBuffers). indexAll's main loop decodes them (or hands the buffers to the store worker), but its two retry passes — plain retry and the comments-stripped last resort — stored the transport as-is: the storage gate passed via errors.length === 0, zero nodes were inserted, and the files row was written with node_count = 0 while the original error was spliced out of the summary. Any worker crash/timeout whose in-flight file was a kernel-routed language permanently recorded that file as "(0 symbols)" — silently, and immune to later syncs because the stored hash matches the on-disk bytes (#1541; v1.4.1 predates the kernel path, which is why it was unaffected). - Both retry passes now materialize kernel results before the gate, store, counters, and log lines. - storeExtractionResult materializes at entry as defense-in-depth, so no storage path can persist an undecoded transport again. - Zero-node rows on symbol-bearing languages (only the wipe produces these — every real extraction stores at least the file node) are dropped during full-reconcile sync and indexAll so already-affected files re-index automatically after upgrading. Scoped watcher syncs leave rows outside their scope untouched. - The comments-stripped salvage now downgrades the failure to a visible warning instead of erasing it: the recovered result can be incomplete, and reporting clean success made a fresh index quietly disagree with a later per-file re-parse of the same bytes (#1565's init-vs-sync divergence). Repro (released 1.5.0): CODEGRAPH_PARSE_TIMEOUT_MS=1 codegraph init on any Python project → "Retry OK: (0 nodes)" and permanent "(python, 0 symbols)" rows. Fixed build stores real symbols under the same forcing, and heals rows wiped by prior runs. Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 2 + __tests__/kernel-retry-materialize.test.ts | 150 +++++++++++++++++++++ src/extraction/index.ts | 59 +++++++- 3 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 __tests__/kernel-retry-materialize.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0485afc..aae12c46a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - When a `codegraph_explore` answer shows a file as excerpts, a large excerpt that no longer fit was dropped entirely instead of being shortened. If the file's first excerpt happened to be a trivial one — an import block, a one-line helper next to the code you asked about — the excerpt carrying the actual answer was the one thrown away, and the file came back with a quarter of the room it had been given. On real projects that meant the top-ranked file delivered a fraction of its share while a far less relevant file took the rest. Excerpts are now shortened to fit, whole method by whole method, and only dropped when what is left is too small to hold anything readable. - When you name a symbol in a `codegraph_explore` query, its definition now actually comes back. Two cases previously lost it. If the symbols you named don't call one another — sibling functions inside the same factory or module are the everyday example — CodeGraph stopped treating them as symbols you had asked for, and answered with whatever sat at the top of their file instead; on one 1,400-line file that meant a same-stem `QueuedMessage` interface on line 70 came back while the `queueMessage` function on line 1087 did not. And when an answer had to be trimmed to fit, it was trimmed from the bottom of the file down, so a symbol near the end of a long file was always the first thing cut. Trimming now protects the definitions you named wherever they sit in the file. - The blast-radius section of `codegraph_explore` flagged "no covering tests found" whenever no test called a symbol directly — falsely branding helpers that tests exercise through their callers as untested (about 40% of flagged symbols in a measured sample). The check now follows caller chains up to 3 hops and reports indirect coverage as "tested via callers"; when nothing is found it states exactly what was checked instead of an unconditional warning. Thanks @inth3shadows for measuring the false-positive rate. (#1475) +- Fixed a v1.5.0 regression where a perfectly valid file could be permanently recorded as having 0 symbols, with no error reported. When a file's first parse attempt was interrupted — a parsing worker crash or timeout, most likely on slow or heavily loaded machines — the automatic retry stored an empty result for any language on the native extraction path, so the file's functions and classes silently vanished from search, callers, and impact until the file was next edited. Retries now store the file's real symbols, and a file already recorded as symbol-free is detected and repaired automatically by the next sync or re-index after upgrading. Thanks @Baiae for the report. (#1541) +- When indexing has to fall back to parsing a file with its comment lines stripped — a last-resort recovery after repeated parser crashes — the file is now flagged with a visible warning instead of being reported as cleanly indexed. The recovered result can be incomplete, and reporting success made a fresh index quietly disagree with a later re-parse of the same unchanged file. Thanks @jeremypetz for the precise init-versus-sync symbol accounting that exposed this. (#1565) ## [1.5.0] - 2026-07-21 diff --git a/__tests__/kernel-retry-materialize.test.ts b/__tests__/kernel-retry-materialize.test.ts new file mode 100644 index 000000000..23965bd2d --- /dev/null +++ b/__tests__/kernel-retry-materialize.test.ts @@ -0,0 +1,150 @@ +/** + * Kernel results must be DECODED before they are persisted (#1541). + * + * The bulk-index parse workers return kernel extractions as an undecoded + * buffer transport: `nodes`/`edges`/`unresolvedReferences` are EMPTY and the + * real tables ride in `kernelBuffers`. The main loop decodes (or hands the + * buffers to the store worker), but indexAll's retry passes used to store the + * transport as-is — the storage gate passed via `errors.length === 0`, zero + * nodes were inserted, and the file was permanently recorded as + * "(0 symbols)" with the retry counted as a success. Any worker + * crash/timeout whose in-flight file was a kernel-routed language silently + * wiped that file's symbols (issue #1541: v1.5.0 indexes a valid Python file + * as 0 symbols; v1.4.1, pre-kernel, indexed it correctly). + * + * This pins the store boundary: storeExtractionResult must materialize a + * buffer-transport result before persisting, so every caller — including the + * retry passes — stores the real nodes. + * + * Skips when no kernel binary is staged (same gating as the parity suites). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { tryKernelExtractRaw } from '../src/extraction/kernel'; +import type { ExtractionResult } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelBuilt = fs.existsSync(KERNEL_PATH); + +describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { + let dir: string; + let cg: CodeGraph; + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kernel-retry-mat-')); + cg = await CodeGraph.init(dir); + await initGrammars(); + await loadGrammarsForLanguages(['python']); + }); + + afterEach(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('storeExtractionResult persists the decoded nodes of a raw kernel result', async () => { + const source = + 'def target_fn(root, mission_path):\n' + + ' return (root, mission_path)\n' + + '\n' + + 'class Adapter:\n' + + ' def adapt(self):\n' + + ' return target_fn(1, 2)\n'; + const filePath = 'adapter.py'; + fs.writeFileSync(path.join(dir, filePath), source); + + // A genuine undecoded transport, exactly as parse-worker builds it. + const raw = tryKernelExtractRaw(filePath, source, 'python'); + expect(raw).not.toBeNull(); + expect(raw!.counts.nodes).toBeGreaterThan(0); + const transport: ExtractionResult = { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: raw!.errors, + durationMs: 0, + kernelBuffers: raw!.buffers, + kernelCounts: raw!.counts, + }; + + const stats = fs.statSync(path.join(dir, filePath)); + const orchestrator = (cg as unknown as { orchestrator: { storeExtractionResult(f: string, c: string, l: string, s: fs.Stats, r: ExtractionResult): Promise } }).orchestrator; + await orchestrator.storeExtractionResult(filePath, source, 'python', stats, transport); + + // The files row must carry the real symbol count, not the transport's + // empty array — a 0 here is the #1541 "(python, 0 symbols)" wipe. + const file = cg.getFile(filePath); + expect(file).not.toBeNull(); + expect(file!.nodeCount).toBe(raw!.counts.nodes); + + // And the nodes themselves must be queryable. + const nodes = cg.getNodesInFile(filePath); + expect(nodes.length).toBe(raw!.counts.nodes); + expect(nodes.map((n) => n.name)).toContain('target_fn'); + expect(nodes.map((n) => n.name)).toContain('Adapter'); + }); +}); + +/** + * Self-heal for rows the released bug already wiped: a files row recorded + * with zero nodes on a symbol-bearing language can only be a #1541 casualty + * (every real extraction stores at least the file node), and its content + * hash matches the on-disk bytes, so hash-based reconciles skip it forever. + * The full-reconcile sync and indexAll now drop such rows so the file + * re-indexes. Kernel-independent — the wipe is simulated at the DB. + */ +describe('zero-node row self-heal (#1541)', () => { + let dir: string; + let cg: CodeGraph; + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'zero-node-heal-')); + fs.writeFileSync( + path.join(dir, 'adapter.py'), + 'def target_fn(root, mission_path):\n' + + ' return (root, mission_path)\n' + + '\n' + + 'class Adapter:\n' + + ' def adapt(self):\n' + + ' return target_fn(1, 2)\n' + ); + cg = await CodeGraph.init(dir); + await cg.indexAll(); + }); + + afterEach(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('sync repairs a wiped row even though the content hash is unchanged', async () => { + const before = cg.getFile('adapter.py'); + expect(before).not.toBeNull(); + expect(before!.nodeCount).toBeGreaterThan(0); + + // Simulate the released-v1.5.0 wipe: nodes gone, row says 0 symbols, + // content hash still matching the file on disk. + const db = (cg as unknown as { db: { getDb(): { prepare(sql: string): { run(...args: unknown[]): unknown } } } }).db.getDb(); + db.prepare('DELETE FROM nodes WHERE file_path = ?').run('adapter.py'); + db.prepare('UPDATE files SET node_count = 0 WHERE path = ?').run('adapter.py'); + expect(cg.getFile('adapter.py')!.nodeCount).toBe(0); + + await cg.sync(); + + const after = cg.getFile('adapter.py'); + expect(after).not.toBeNull(); + expect(after!.nodeCount).toBe(before!.nodeCount); + expect(cg.getNodesInFile('adapter.py').map((n) => n.name)).toContain('target_fn'); + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 22108d1d1..9e915a247 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -1577,6 +1577,11 @@ export class ExtractionOrchestrator { }); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`); + // A re-index over an existing DB skips unchanged-hash files at the store, + // which would preserve wiped zero-node rows (#1541) — drop them first so + // this run stores their files fresh. No-op on a fresh DB. + this.healZeroNodeRows(); + // Detect frameworks once per indexAll run using the scanned file list. // Names are passed to each parse call so framework-specific extractors // (route nodes, middleware, etc.) run after the tree-sitter pass. @@ -2025,8 +2030,16 @@ export class ExtractionOrchestrator { continue; } + // The pool hands kernel results back as an undecoded buffer transport + // (`nodes`/`edges` EMPTY, tables in kernelBuffers). The main loop + // decodes or forwards to the store worker; this path stores directly, + // so decode here — otherwise a kernel-language retry passes the gate + // below via `errors.length === 0`, stores nothing, and the file is + // permanently recorded as "(0 symbols)" with the error erased (#1541). + const language = detectLanguage(filePath, content, overrides); + result = materializeKernelResult(result, filePath, language); + if (result.nodes.length > 0 || result.errors.length === 0) { - const language = detectLanguage(filePath, content, overrides); const stats = await fsp.stat(path.join(this.rootDir, filePath)); await this.storeExtractionResult(filePath, content, language, stats, result, commitYield); @@ -2075,13 +2088,21 @@ export class ExtractionOrchestrator { continue; } + // Same undecoded-transport hazard as the first retry pass (#1541). + const language = detectLanguage(filePath, fullContent, overrides); + result = materializeKernelResult(result, filePath, language); + if (result.nodes.length > 0 || result.errors.length === 0) { - const language = detectLanguage(filePath, fullContent, overrides); const stats = await fsp.stat(path.join(this.rootDir, filePath)); await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield); - const idx = errors.indexOf(errEntry); - if (idx >= 0) errors.splice(idx, 1); + // Salvaged from comment-stripped source: keep a visible trace in + // the summary instead of erasing the failure outright — the + // stored result may be missing whatever the failing parse choked + // on, and a silently "clean" file here is how an index quietly + // disagrees with a later per-file sync of the same bytes (#1565). + errEntry.severity = 'warning'; + errEntry.message = `Indexed from comment-stripped source after repeated parse failures (symbols may be incomplete until the file is re-indexed): ${errEntry.message}`; filesErrored--; filesIndexed++; totalNodes += result.nodes.length; @@ -2267,6 +2288,26 @@ export class ExtractionOrchestrator { /** * Store extraction result in database */ + /** + * Delete file rows recorded with ZERO nodes so their files re-index. + * + * No extraction path stores an empty, error-free result for a + * symbol-bearing language — even an empty file keeps its file node — so a + * zero-node row is a wiped one (#1541: an interrupted parse's retry stored + * an undecoded kernel transport). The wiped row's content hash matches the + * on-disk bytes, so every hash-based reconcile skips the file forever; + * deleting the row lets the normal add path repair it. File-level-only + * languages (yaml, twig, properties) are left alone. Deleting a zero-node + * row cascades nothing: it has no nodes, so no edges or refs either. + */ + private healZeroNodeRows(): void { + for (const f of this.queries.getAllFiles()) { + if (f.nodeCount === 0 && !isFileLevelOnlyLanguage(f.language)) { + this.queries.deleteFile(f.path); + } + } + } + private async storeExtractionResult( filePath: string, content: string, @@ -2275,6 +2316,12 @@ export class ExtractionOrchestrator { result: ExtractionResult, onYield?: MaybeYield ): Promise { + // A kernel result can arrive as an undecoded buffer transport (empty + // node/edge arrays, tables riding in kernelBuffers). Decode it before + // storing — persisting the transport as-is records the file as having no + // symbols at all (#1541). No-op for already-decoded results. + result = materializeKernelResult(result, filePath, language); + // Bulk inserts run in bounded sub-transactions with a yield between, so a // giant generated file (tens of thousands of symbols) can't block the // event loop — and the #850 watchdog heartbeat — for the whole store. @@ -2636,6 +2683,10 @@ export class ExtractionOrchestrator { if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`); filesChecked = currentFiles.length; + // Full reconcile only (scoped syncs must not touch rows outside their + // scope): drop zero-node rows so the wiped files re-index as adds below. + this.healZeroNodeRows(); + const tTracked = Date.now(); trackedFiles = this.queries.getAllFiles(); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`); From d8f2eeaddffd8c993f727d57bb621c8ffbc52abc Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Thu, 20 Aug 2026 12:36:22 -0500 Subject: [PATCH 04/27] fix(db): loop-append dense unresolved-ref result rows; make stripped-salvage visible (#1558) (#1576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-world validation of #1575 on indexes damaged by the released v1.5.0 binary surfaced both of these. getUnresolvedReferencesByFiles chunked its INPUT under SQLite's parameter limit but appended each chunk's RESULT rows with a spread — every row becomes a call argument, so a dense recovery sync (the #1541 self-heal re-indexing 919 files produced 234,440 rows) exceeded V8's argument limit and killed resolution mid-sync with "Maximum call stack size exceeded", leaving the graph 226k edges short until another sync resumed the orphans (and that sweep resolves measurably worse than the batched path — see the follow-up issue). The failed-ref retry loader had the identical pattern on unbounded result rows. Both append with a loop now (#1558). The #1575 stripped-salvage warning also never rendered: init's summary prints only index_partial warnings and counts only hard errors, so a run with salvaged files still read as fully clean — and with no hard errors the detail wasn't written to errors.log either. Salvage entries now carry code 'salvaged_stripped', the summary prints a visible warning naming the files, and errors.log is written for salvage-only runs. Validated on real corpora with full-graph dumps: healthy-path inits stay byte-identical to the pre-#1575 baseline (cpython Lib, Alamofire, with a determinism control); a realistically-damaged index (41 wiped + 5 missing files, damage generated by the released binary) heals in one plain sync to identical per-file counts and an edge set within the normal incremental residual; pathological mass damage (52% of the repo) completes without crashing. New regression test reproduces the RangeError on the old code with 200k pending refs. Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 1 + __tests__/refs-by-files-spread.test.ts | 74 ++++++++++++++++++++++++++ src/bin/codegraph.ts | 23 ++++++-- src/db/queries.ts | 12 ++++- src/extraction/index.ts | 1 + 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 __tests__/refs-by-files-spread.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index aae12c46a..32724a53a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - The blast-radius section of `codegraph_explore` flagged "no covering tests found" whenever no test called a symbol directly — falsely branding helpers that tests exercise through their callers as untested (about 40% of flagged symbols in a measured sample). The check now follows caller chains up to 3 hops and reports indirect coverage as "tested via callers"; when nothing is found it states exactly what was checked instead of an unconditional warning. Thanks @inth3shadows for measuring the false-positive rate. (#1475) - Fixed a v1.5.0 regression where a perfectly valid file could be permanently recorded as having 0 symbols, with no error reported. When a file's first parse attempt was interrupted — a parsing worker crash or timeout, most likely on slow or heavily loaded machines — the automatic retry stored an empty result for any language on the native extraction path, so the file's functions and classes silently vanished from search, callers, and impact until the file was next edited. Retries now store the file's real symbols, and a file already recorded as symbol-free is detected and repaired automatically by the next sync or re-index after upgrading. Thanks @Baiae for the report. (#1541) - When indexing has to fall back to parsing a file with its comment lines stripped — a last-resort recovery after repeated parser crashes — the file is now flagged with a visible warning instead of being reported as cleanly indexed. The recovered result can be incomplete, and reporting success made a fresh index quietly disagree with a later re-parse of the same unchanged file. Thanks @jeremypetz for the precise init-versus-sync symbol accounting that exposed this. (#1565) +- Syncing a large batch of changed files no longer crashes with "Maximum call stack size exceeded" partway through. The crash aborted reference resolution after the files' symbols were already stored, leaving the graph with far fewer connections than a fresh index would have — and it hit exactly the scenarios that re-parse many files at once, including the automatic repair above. Thanks @netbrah for pinpointing the failure. (#1558) ## [1.5.0] - 2026-07-21 diff --git a/__tests__/refs-by-files-spread.test.ts b/__tests__/refs-by-files-spread.test.ts new file mode 100644 index 000000000..7d93429fb --- /dev/null +++ b/__tests__/refs-by-files-spread.test.ts @@ -0,0 +1,74 @@ +/** + * getUnresolvedReferencesByFiles must survive dense result sets (#1558). + * + * The input file-path list is chunked under SQLite's parameter limit, but the + * ROWS a chunk returns are unbounded — and appending them with + * `rows.push(...chunkRows)` passes every row as a call argument, so a dense + * chunk (a recovery sync re-indexing many files at once, e.g. the #1541 + * self-heal) exceeded V8's argument limit and killed the whole sync with + * "Maximum call stack size exceeded" after the store phase, leaving every + * re-indexed file's references unresolved. Reproduced for real on a + * cpython-stdlib-sized heal (919 files, 234k refs). The append is now a loop; + * this pins it with a result set well past V8's argument ceiling (~124k). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; +import type { UnresolvedReference } from '../src/types'; + +describe('unresolved-ref loads with dense result sets (#1558)', () => { + let dir: string; + let cg: CodeGraph; + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'refs-spread-')); + fs.writeFileSync(path.join(dir, 'anchor.py'), 'def anchor():\n return 1\n'); + cg = await CodeGraph.init(dir); + await cg.indexAll(); + }); + + afterEach(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('returns 200k pending refs from few files without exhausting the call stack', () => { + const queries = (cg as unknown as { + queries: { + insertUnresolvedRefsBatch(refs: UnresolvedReference[]): void; + getUnresolvedReferencesByFiles(paths: string[]): UnresolvedReference[]; + }; + }).queries; + + const FILES = 200; + const TOTAL = 200_000; + const paths: string[] = Array.from({ length: FILES }, (_, i) => `src/f${i}.py`); + // unresolved_refs.from_node_id is FK-constrained — anchor on a real node. + const anchorId = cg.getNodesInFile('anchor.py')[0]!.id; + + const batch: UnresolvedReference[] = []; + for (let i = 0; i < TOTAL; i++) { + batch.push({ + fromNodeId: anchorId, + referenceName: `ref_${i}`, + referenceKind: 'call', + line: (i % 1000) + 1, + column: 0, + filePath: paths[i % FILES]!, + language: 'python', + }); + if (batch.length === 20_000) { + queries.insertUnresolvedRefsBatch(batch); + batch.length = 0; + } + } + if (batch.length > 0) queries.insertUnresolvedRefsBatch(batch); + + // All 200 paths fit in ONE SQLite parameter chunk, so a single query + // returns all 200k rows — the exact shape that blew the argument limit. + const rows = queries.getUnresolvedReferencesByFiles(paths); + expect(rows.length).toBe(TOTAL); + }); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 067cdd3e0..acec9c2cb 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -405,6 +405,16 @@ function printIndexResult(clack: typeof import('@clack/prompts'), result: IndexR for (const w of result.errors.filter((e) => e.code === 'index_partial')) { clack.log.warn(w.message); } + // Files salvaged from comment-stripped source after repeated parser + // failures are indexed but possibly incomplete — say so here, or the run + // reads as fully clean and the index quietly disagrees with a later + // re-parse of the same bytes (#1565). + const salvaged = result.errors.filter((e) => e.code === 'salvaged_stripped'); + if (salvaged.length > 0) { + const sample = salvaged.slice(0, 3).map((e) => e.filePath).filter(Boolean).join(', '); + const more = salvaged.length > 3 ? ', ...' : ''; + clack.log.warn(`${formatNumber(salvaged.length)} file(s) indexed from comment-stripped source after repeated parse failures ${getGlyphs().dash} symbols may be incomplete (${sample}${more})`); + } } else if (hasErrors) { clack.log.error(`Indexing failed ${getGlyphs().dash} all ${formatNumber(result.filesErrored)} files had errors`); } else { @@ -443,9 +453,16 @@ function printIndexResult(clack: typeof import('@clack/prompts'), result: IndexR clack.log.info(`The index is fully usable ${getGlyphs().dash} only the failed files are missing.`); } } else if (projectPath) { - const logPath = path.join(getCodeGraphDir(projectPath), 'errors.log'); - if (fs.existsSync(logPath)) { - fs.unlinkSync(logPath); + // No hard errors. Salvaged-file warnings still belong in the log — it + // carries the per-file detail behind the one-line summary above. + if (result.errors.some((e) => e.code === 'salvaged_stripped')) { + writeErrorLog(projectPath, result.errors); + clack.log.info('See .codegraph/errors.log for details'); + } else { + const logPath = path.join(getCodeGraphDir(projectPath), 'errors.log'); + if (fs.existsSync(logPath)) { + fs.unlinkSync(logPath); + } } } } diff --git a/src/db/queries.ts b/src/db/queries.ts index 451c6a90f..2b8bc5344 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -2359,7 +2359,12 @@ export class QueryBuilder { const chunkRows = this.db .prepare(`SELECT * FROM unresolved_refs WHERE status = 'pending' AND file_path IN (${placeholders})`) .all(...chunk) as UnresolvedRefRow[]; - rows.push(...chunkRows); + // Append with a loop, never a spread: the INPUT chunk is bounded, but + // the RESULT rows per chunk are not — a dense recovery sync (e.g. the + // #1541 self-heal re-indexing hundreds of files) returns more rows than + // V8 allows as arguments, and `push(...chunkRows)` dies with "Maximum + // call stack size exceeded", aborting resolution mid-sync (#1558). + for (const row of chunkRows) rows.push(row); } return rows.map((row) => ({ @@ -2541,7 +2546,10 @@ export class QueryBuilder { const chunkRows = this.db .prepare(`SELECT * FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders})`) .all(...chunk) as UnresolvedRefRow[]; - rows.push(...chunkRows); + // Loop, not spread — same V8 argument-limit hazard as + // getUnresolvedReferencesByFiles (#1558): a large definition delta can + // select an unbounded number of failed rows per chunk. + for (const row of chunkRows) rows.push(row); } return rows.map((row) => ({ diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 9e915a247..3606d2591 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -2102,6 +2102,7 @@ export class ExtractionOrchestrator { // on, and a silently "clean" file here is how an index quietly // disagrees with a later per-file sync of the same bytes (#1565). errEntry.severity = 'warning'; + errEntry.code = 'salvaged_stripped'; errEntry.message = `Indexed from comment-stripped source after repeated parse failures (symbols may be incomplete until the file is re-indexed): ${errEntry.message}`; filesErrored--; filesIndexed++; From 81e1f4a92fdbd9413ba73cf72c8a5408102a7897 Mon Sep 17 00:00:00 2001 From: Daniil Date: Thu, 20 Aug 2026 20:53:49 +0300 Subject: [PATCH 05/27] fix: harden daemon and large-index recovery paths (#1562) * fix: harden indexing recovery and daemon liveness * test: cover daemon and recovery review gaps * test: pin that a failure marker never blocks a later successful parse (#1557 retry-discard guard) Co-Authored-By: Claude Fable 5 --------- Co-authored-by: danusha2345 Co-authored-by: Colby McHenry Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 6 + __tests__/cli-unlock.test.ts | 102 +++++++++++++++ __tests__/daemon-registry.test.ts | 55 ++++++++ __tests__/foundation.test.ts | 35 +++++ __tests__/large-corpus-regressions.test.ts | 143 +++++++++++++++++++++ __tests__/mcp-daemon.test.ts | 39 ++++++ __tests__/parse-pool.test.ts | 13 +- __tests__/sync.test.ts | 22 ++++ src/bin/codegraph.ts | 23 ++-- src/db/index.ts | 23 ++++ src/extraction/index.ts | 117 ++++++++++------- src/extraction/parse-pool.ts | 15 ++- src/index.ts | 8 ++ src/mcp/daemon-manager.ts | 4 +- src/mcp/daemon-paths.ts | 50 +++++++ src/mcp/daemon-registry.ts | 56 +++++++- src/mcp/daemon.ts | 22 ++-- src/mcp/index.ts | 19 ++- src/resolution/c-fnptr-synthesizer.ts | 4 +- src/resolution/callback-synthesizer.ts | 9 +- 20 files changed, 681 insertions(+), 84 deletions(-) create mode 100644 __tests__/cli-unlock.test.ts create mode 100644 __tests__/large-corpus-regressions.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 32724a53a..f44be8edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,12 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Fixed a v1.5.0 regression where a perfectly valid file could be permanently recorded as having 0 symbols, with no error reported. When a file's first parse attempt was interrupted — a parsing worker crash or timeout, most likely on slow or heavily loaded machines — the automatic retry stored an empty result for any language on the native extraction path, so the file's functions and classes silently vanished from search, callers, and impact until the file was next edited. Retries now store the file's real symbols, and a file already recorded as symbol-free is detected and repaired automatically by the next sync or re-index after upgrading. Thanks @Baiae for the report. (#1541) - When indexing has to fall back to parsing a file with its comment lines stripped — a last-resort recovery after repeated parser crashes — the file is now flagged with a visible warning instead of being reported as cleanly indexed. The recovered result can be incomplete, and reporting success made a fresh index quietly disagree with a later re-parse of the same unchanged file. Thanks @jeremypetz for the precise init-versus-sync symbol accounting that exposed this. (#1565) - Syncing a large batch of changed files no longer crashes with "Maximum call stack size exceeded" partway through. The crash aborted reference resolution after the files' symbols were already stored, leaving the graph with far fewer connections than a fresh index would have — and it hit exactly the scenarios that re-parse many files at once, including the automatic repair above. Thanks @netbrah for pinpointing the failure. (#1558) +- A background daemon left behind by an out-of-memory kill or force-kill can no longer block every future session when the operating system reuses its process ID: daemon management now verifies a recorded process is really a CodeGraph daemon before trusting or signaling it, and `codegraph unlock` clears stale daemon artifacts as well as the indexing lock. Thanks @hcg1023 for the report and @danusha2345 for the fix. (#1553) +- Data-only C/C++ headers near the file-size limit no longer hold a parser worker for several minutes before timing out; the default large-file parse budget is now bounded, while an explicitly configured larger timeout is still honored. (#1555) +- Reopening an index after a crash during bulk loading now restores every dropped database index, and a successful recovery sync marks the index complete instead of leaving it permanently flagged as interrupted. (#1556) +- Files skipped because they are too large or repeatedly fail to parse are now recorded with the reason, so unchanged rejected files are no longer rediscovered and retried on every status check and sync — and a later successful parse of such a file replaces the record with its real symbols. Thanks @netbrah for the exceptional failure analysis behind this batch, and @danusha2345 for the fixes. (#1557) +- C/C++ function-pointer analysis now bounds its compiled-pattern caches, so very large repositories can no longer exhaust the JavaScript engine's regular-expression code space during indexing. (#1559) +- JSX rendering analysis now runs only on JavaScript-family files, so JSX-looking strings in C/C++ (or any other language) no longer create impossible call edges — in pure-C projects and in mixed-language monorepos alike. (#1560) ## [1.5.0] - 2026-07-21 diff --git a/__tests__/cli-unlock.test.ts b/__tests__/cli-unlock.test.ts new file mode 100644 index 000000000..9db3b7a02 --- /dev/null +++ b/__tests__/cli-unlock.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { execFile, execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as net from 'net'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { getDaemonPidPath, getDaemonSocketPath } from '../src/mcp/daemon-paths'; +import { CodeGraphPackageVersion } from '../src/mcp/version'; + +const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); + +function runCodegraph(args: string[], cwd: string): string { + return execFileSync(process.execPath, [BIN, ...args], { + cwd, + encoding: 'utf8', + env: { ...process.env, CODEGRAPH_NO_DAEMON: '1' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function runCodegraphAsync(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + execFile( + process.execPath, + [BIN, ...args], + { cwd, encoding: 'utf8', env: { ...process.env, CODEGRAPH_NO_DAEMON: '1' } }, + (error, stdout, stderr) => { + if (error) reject(new Error(`${error.message}\n${stderr}`)); + else resolve(stdout); + }, + ); + }); +} + +describe('codegraph unlock — daemon artifact recovery (#1553)', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-unlock-')); + const cg = CodeGraph.initSync(tempDir); + cg.close(); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('removes indexing and phantom-daemon artifacts, then permits indexing', () => { + const graphDir = path.join(tempDir, '.codegraph'); + const pidPath = getDaemonPidPath(tempDir); + const socketPath = getDaemonSocketPath(tempDir); + fs.writeFileSync(path.join(graphDir, 'codegraph.lock'), 'stale\n'); + fs.writeFileSync(pidPath, JSON.stringify({ + pid: process.pid, + version: CodeGraphPackageVersion, + socketPath, + startedAt: Date.now() - 60_000, + })); + if (process.platform !== 'win32') fs.writeFileSync(socketPath, 'stale\n'); + + const output = runCodegraph(['unlock', tempDir], tempDir); + + expect(output).toContain('Removed stale lock artifacts'); + expect(fs.existsSync(path.join(graphDir, 'codegraph.lock'))).toBe(false); + expect(fs.existsSync(pidPath)).toBe(false); + if (process.platform !== 'win32') expect(fs.existsSync(socketPath)).toBe(false); + expect(() => process.kill(process.pid, 0)).not.toThrow(); + expect(() => runCodegraph(['index', '--quiet', tempDir], tempDir)).not.toThrow(); + }); + + it('preserves artifacts when the recorded live daemon answers the socket hello', async () => { + const pidPath = getDaemonPidPath(tempDir); + const socketPath = getDaemonSocketPath(tempDir); + const server = net.createServer((socket) => { + socket.end(JSON.stringify({ + codegraph: CodeGraphPackageVersion, + pid: process.pid, + socketPath, + protocol: 1, + }) + '\n'); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, resolve); + }); + fs.writeFileSync(pidPath, JSON.stringify({ + pid: process.pid, + version: CodeGraphPackageVersion, + socketPath, + startedAt: Date.now(), + })); + + try { + const output = await runCodegraphAsync(['unlock', tempDir], tempDir); + expect(output).toContain('No stale lock files found'); + expect(fs.existsSync(pidPath)).toBe(true); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); diff --git a/__tests__/daemon-registry.test.ts b/__tests__/daemon-registry.test.ts index 55bafc45a..aa13ec100 100644 --- a/__tests__/daemon-registry.test.ts +++ b/__tests__/daemon-registry.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { spawn } from 'child_process'; import * as fs from 'fs'; +import * as net from 'net'; import * as os from 'os'; import * as path from 'path'; import { @@ -9,8 +10,11 @@ import { registerDaemon, deregisterDaemon, listDaemons, + listVerifiedDaemons, + stopDaemonAt, type DaemonRecord, } from '../src/mcp/daemon-registry'; +import { encodeLockInfo, getDaemonPidPath } from '../src/mcp/daemon-paths'; /** A pid that's guaranteed dead: spawn a trivial process, let it exit, reap it. */ async function deadPid(): Promise { @@ -100,4 +104,55 @@ describe('daemon-registry', () => { const live = listDaemons(); expect(live.map((d) => d.root)).toEqual(['/proj/new', '/proj/old']); }); + + it('keeps a registry entry whose socket hello matches its PID and version', async () => { + const root = fs.mkdtempSync(path.join(tmpHome, 'verified-')); + const socketPath = process.platform === 'win32' + ? `\\\\.\\pipe\\cg-reg-${process.pid}-${Date.now()}` + : path.join(tmpHome, 'verified.sock'); + const server = net.createServer((socket) => { + socket.end(JSON.stringify({ + protocol: 1, + pid: process.pid, + codegraph: '1.5.0', + socketPath, + }) + '\n'); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, resolve); + }); + try { + registerDaemon({ root, pid: process.pid, version: '1.5.0', socketPath, startedAt: 1 }); + expect((await listVerifiedDaemons()).map((d) => d.root)).toEqual([root]); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it('never signals a reused live PID when no matching daemon answers (#1553)', async () => { + const root = fs.mkdtempSync(path.join(tmpHome, 'project-')); + const pidPath = getDaemonPidPath(root); + fs.mkdirSync(path.dirname(pidPath), { recursive: true }); + fs.writeFileSync(pidPath, encodeLockInfo({ + pid: process.pid, + version: '1.5.0', + socketPath: path.join(root, '.codegraph', 'missing.sock'), + startedAt: Date.now() - 60_000, + })); + + registerDaemon({ + root, + pid: process.pid, + version: '1.5.0', + socketPath: path.join(root, '.codegraph', 'missing.sock'), + startedAt: Date.now() - 60_000, + }); + + expect(await listVerifiedDaemons()).toEqual([]); + const result = await stopDaemonAt(root); + expect(result).toMatchObject({ pid: process.pid, outcome: 'not-running' }); + expect(isProcessAlive(process.pid)).toBe(true); + expect(fs.existsSync(pidPath)).toBe(false); + }); }); diff --git a/__tests__/foundation.test.ts b/__tests__/foundation.test.ts index 12c136445..b7616272a 100644 --- a/__tests__/foundation.test.ts +++ b/__tests__/foundation.test.ts @@ -120,6 +120,41 @@ describe('CodeGraph Foundation', () => { cg.close(); }); + it('restores every secondary index after a crash inside bulk parse load (#1556)', () => { + const dbPath = getDatabasePath(tempDir); + const first = DatabaseConnection.initialize(dbPath); + const before = (first.getDb() + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' ORDER BY name") + .all() as Array<{ name: string }>).map((r) => r.name); + first.beginBulkParseLoad(); + first.close(); + + const reopened = DatabaseConnection.open(dbPath); + const after = (reopened.getDb() + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' ORDER BY name") + .all() as Array<{ name: string }>).map((r) => r.name); + reopened.close(); + + expect(after).toEqual(before); + }); + + it('skips secondary-index DDL when the schema is already healthy', () => { + const dbPath = getDatabasePath(tempDir); + const connection = DatabaseConnection.initialize(dbPath); + const db = connection.getDb(); + const originalExec = db.exec.bind(db); + let execCalls = 0; + db.exec = (sql: string) => { + execCalls++; + originalExec(sql); + }; + + (connection as any).healBulkSecondaryIndexes(); + connection.close(); + + expect(execCalls).toBe(0); + }); + it('should return correct database size', () => { const cg = CodeGraph.initSync(tempDir); const stats = cg.getStats(); diff --git a/__tests__/large-corpus-regressions.test.ts b/__tests__/large-corpus-regressions.test.ts new file mode 100644 index 000000000..109032cb8 --- /dev/null +++ b/__tests__/large-corpus-regressions.test.ts @@ -0,0 +1,143 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import CodeGraph from '../src/index'; +import { QueryBuilder } from '../src/db/queries'; + +describe('large-corpus regression fixes', () => { + it('collects a dense unresolved-reference chunk without spreading it onto the V8 stack (#1558)', () => { + const row = { + id: 1, + from_node_id: 'source', + reference_name: 'target', + reference_kind: 'calls', + line: 1, + col: 1, + candidates: null, + file_path: 'dense.c', + language: 'c', + status: 'pending', + name_tail: 'target', + }; + const denseRows = new Array(200_000).fill(row); + const db = { prepare: () => ({ all: () => denseRows }) }; + const queries = new QueryBuilder(db as any); + expect(queries.getUnresolvedReferencesByFiles(['dense.c'])).toHaveLength(200_000); + }); + + it('records an oversized file during a fresh index so sync does not retry it (#1557)', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-skipped-file-')); + try { + fs.writeFileSync(path.join(dir, 'oversized.py'), 'value = 1\n'.repeat(120_000)); + const cg = await CodeGraph.init(dir, { silent: true }); + const indexed = await cg.indexAll(); + expect(indexed.filesSkipped).toBe(1); + expect(cg.getFiles().find((f) => f.path === 'oversized.py')?.errors?.[0]?.code).toBe('size_exceeded'); + const synced = await cg.sync(); + expect(synced.filesAdded).toBe(0); + cg.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('records an oversized file through the single-file indexing path (#1557)', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-single-skipped-file-')); + try { + fs.writeFileSync(path.join(dir, 'oversized.py'), 'value = 1\n'.repeat(120_000)); + const cg = await CodeGraph.init(dir, { silent: true }); + const indexed = await cg.indexFiles(['oversized.py']); + expect(indexed.filesSkipped).toBe(1); + expect(cg.getFiles().find((f) => f.path === 'oversized.py')?.errors?.[0]?.code).toBe('size_exceeded'); + const synced = await cg.sync(); + expect(synced.filesAdded).toBe(0); + expect(synced.filesModified).toBe(0); + cg.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('JSX synthesis language boundary (#1560)', () => { + let dir: string; + beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-jsx-gate-')); }); + afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); }); + + it('does not create jsx-render edges from JSX-looking text in a C-only project', async () => { + fs.writeFileSync( + path.join(dir, 'only.c'), + 'void Foo(void) {}\nvoid parent(void) { const char *s = ""; }\n' + ); + const cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + const rows = (cg as any).db.db.prepare( + "SELECT count(*) AS c FROM edges WHERE json_extract(metadata, '$.synthesizedBy') = 'jsx-render'" + ).get() as { c: number }; + cg.close(); + expect(rows.c).toBe(0); + }); + + it('runs for JavaScript while excluding C parents in the same project', async () => { + fs.writeFileSync( + path.join(dir, 'native.c'), + 'void Widget(void) {}\nvoid native_parent(void) { const char *s = ""; }\n' + ); + fs.writeFileSync( + path.join(dir, 'ui.jsx'), + 'export function Widget() { return ; }\nexport function App() { return ; }\n' + ); + const cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + const rows = (cg as any).db.db.prepare(` + SELECT source.file_path AS source_file, target.name AS target_name + FROM edges e + JOIN nodes source ON source.id = e.source + JOIN nodes target ON target.id = e.target + WHERE json_extract(e.metadata, '$.synthesizedBy') = 'jsx-render' + `).all() as Array<{ source_file: string; target_name: string }>; + cg.close(); + expect(rows).toContainEqual({ source_file: 'ui.jsx', target_name: 'Widget' }); + expect(rows.some((row) => row.source_file === 'native.c')).toBe(false); + }); +}); + +describe('failure markers vs later real results (#1557 × #1541)', () => { + it('a failure marker never blocks storing a later successful parse of the same bytes', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-marker-override-')); + try { + const rel = 'flaky.py'; + const content = 'def real_fn():\n return 1\n\nclass RealClass:\n def m(self):\n return 2\n'; + fs.writeFileSync(path.join(dir, rel), content); + const cg = await CodeGraph.init(dir, { silent: true }); + const { initGrammars, loadGrammarsForLanguages } = await import('../src/extraction/grammars'); + await initGrammars(); + await loadGrammarsForLanguages(['python']); + const orch = (cg as any).orchestrator; + const stats = fs.statSync(path.join(dir, rel)); + + // What recordParseFailure persists when a parse worker dies: a marker + // row under the SAME content hash the retry will store with. + await orch.storeExtractionResult(rel, content, 'python', stats, { + nodes: [], edges: [], unresolvedReferences: [], + errors: [{ message: 'Worker exited with code 1', filePath: rel, severity: 'error', code: 'parse_error' }], + durationMs: 0, + }); + expect(cg.getFile(rel)?.nodeCount).toBe(0); + + // The retry pass succeeds with identical bytes — the marker must be + // replaced, not treated as "no changes". + const { extractFromSource } = await import('../src/extraction/tree-sitter'); + const real = extractFromSource(rel, content, 'python'); + expect(real.nodes.length).toBeGreaterThan(0); + await orch.storeExtractionResult(rel, content, 'python', stats, real); + + expect(cg.getFile(rel)?.nodeCount).toBe(real.nodes.length); + expect(cg.getNodesInFile(rel).map((n: { name: string }) => n.name)).toContain('real_fn'); + cg.close(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/__tests__/mcp-daemon.test.ts b/__tests__/mcp-daemon.test.ts index ab7613664..c73ac564c 100644 --- a/__tests__/mcp-daemon.test.ts +++ b/__tests__/mcp-daemon.test.ts @@ -39,6 +39,7 @@ import * as os from 'os'; import * as path from 'path'; import { CodeGraph } from '../src'; import { getDaemonSocketPath } from '../src/mcp/daemon-paths'; +import { CodeGraphPackageVersion } from '../src/mcp/version'; const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); @@ -336,6 +337,44 @@ describe('Shared MCP daemon (issue #411)', () => { expect(isAlive(livePid!)).toBe(true); }, 40000); + it('takes over after SIGKILL even when the stale PID has been reused (#1553)', async () => { + const env = { CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS: '30000' }; + const first = spawnServer(tempDir, env); + servers.push(first); + sendInitialize(first.child, `file://${tempDir}`, 1); + await waitFor(() => findResponse(first.stdout, 1), 10000); + await waitFor(() => countListeningLines(realRoot) >= 1, 10000); + const killedPid = readLockPid(realRoot)!; + + process.kill(killedPid, 'SIGKILL'); + expect(await waitProcessExit(killedPid, 8000)).toBe(true); + + // Model OS PID reuse without risking another process: the stale lock now + // names this live vitest worker, but no daemon answers the leftover socket. + fs.writeFileSync( + path.join(realRoot, '.codegraph', 'daemon.pid'), + JSON.stringify({ + pid: process.pid, + version: CodeGraphPackageVersion, + socketPath: getDaemonSocketPath(realRoot), + startedAt: Date.now() - 60_000, + }), + ); + + const second = spawnServer(tempDir, env); + servers.push(second); + sendInitialize(second.child, `file://${tempDir}`, 2); + const response = await waitFor(() => findResponse(second.stdout, 2), 12000); + expect(response.result.serverInfo.name).toBe('codegraph'); + await waitFor(() => countListeningLines(realRoot) >= 2, 10000); + + const replacementPid = readLockPid(realRoot)!; + expect(replacementPid).not.toBe(killedPid); + expect(replacementPid).not.toBe(process.pid); + expect(isAlive(replacementPid)).toBe(true); + expect(isAlive(process.pid)).toBe(true); + }, 50000); + it('proxy falls back to direct mode on a daemon version mismatch', async () => { const net = await import('net'); const sockPath = getDaemonSocketPath(realRoot); diff --git a/__tests__/parse-pool.test.ts b/__tests__/parse-pool.test.ts index 641d24d12..6211481a4 100644 --- a/__tests__/parse-pool.test.ts +++ b/__tests__/parse-pool.test.ts @@ -11,7 +11,7 @@ * parallelism safe. */ import { describe, it, expect } from 'vitest'; -import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs, type ParsePoolWorker, type ParseTask } from '../src/extraction/parse-pool'; +import { ParseWorkerPool, resolveParseBudgetMs, resolveParsePoolSize, resolveParseTimeoutMs, type ParsePoolWorker, type ParseTask } from '../src/extraction/parse-pool'; import type { Language, ExtractionResult } from '../src/types'; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -94,6 +94,17 @@ describe('resolveParseTimeoutMs', () => { }); }); +describe('resolveParseBudgetMs', () => { + it('caps near-limit blob headers at a 20s soft / 60s hard window (#1555)', () => { + expect(resolveParseBudgetMs(10_000, 940_800)).toBe(20_000); + expect(resolveParseBudgetMs(10_000, 857_376)).toBe(20_000); + }); + + it('does not clamp an explicit larger base timeout', () => { + expect(resolveParseBudgetMs(45_000, 940_800)).toBe(45_000); + }); +}); + describe('resolveParsePoolSize', () => { it('treats explicit 0 and 1 as a single worker (the rollback path)', () => { expect(resolveParsePoolSize('0', 8)).toBe(1); diff --git a/__tests__/sync.test.ts b/__tests__/sync.test.ts index f26c05e1f..c85877c80 100644 --- a/__tests__/sync.test.ts +++ b/__tests__/sync.test.ts @@ -149,6 +149,28 @@ describe('Sync Module', () => { expect(result.filesRemoved).toBe(0); expect(result.filesChecked).toBeGreaterThan(0); }); + + it('persists an oversized skipped file so later syncs do not retry it (#1557)', async () => { + const filePath = path.join(testDir, 'src', 'oversized.ts'); + fs.writeFileSync(filePath, 'const value = 1;\n'.repeat(70_000)); + + const first = await cg.sync(); + expect(first.filesAdded).toBe(1); + expect(cg.getFiles().find((f) => f.path === 'src/oversized.ts')?.errors?.[0]?.code).toBe('size_exceeded'); + + const second = await cg.sync(); + expect(second.filesAdded).toBe(0); + expect(second.filesModified).toBe(0); + }); + + it('marks a successfully recovered indexing state complete (#1556)', async () => { + (cg as any).queries.setMetadata('index_state', 'indexing'); + await cg.sync({ paths: ['src/index.ts'] }); + expect(cg.getIndexState()).toBe('indexing'); + + await cg.sync(); + expect(cg.getIndexState()).toBe('complete'); + }); }); }); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index acec9c2cb..e4038200d 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -1708,10 +1708,10 @@ program .aliases(['daemons']) .description('Manage running CodeGraph background daemons — pick one and press enter to stop it') .action(async () => { - const { listDaemons, stopDaemonAt, stopAllDaemons } = await import('../mcp/daemon-registry'); + const { listVerifiedDaemons, stopDaemonAt, stopAllDaemons } = await import('../mcp/daemon-registry'); const { runDaemonPicker } = await import('../mcp/daemon-manager'); - const daemons = listDaemons(); + const daemons = await listVerifiedDaemons(); if (daemons.length === 0) { info('No CodeGraph daemons running.'); return; @@ -1734,7 +1734,7 @@ program const clack = await importESM('@clack/prompts'); clack.intro('CodeGraph daemons'); await runDaemonPicker({ - list: listDaemons, + list: listVerifiedDaemons, stop: stopDaemonAt, stopAll: stopAllDaemons, cwdRoot, @@ -1840,14 +1840,15 @@ program } const lockPath = path.join(getCodeGraphDir(projectPath), 'codegraph.lock'); - - if (!fs.existsSync(lockPath)) { - info(`No lock file found ${getGlyphs().dash} nothing to do`); - return; - } - - fs.unlinkSync(lockPath); - success('Removed lock file. You can now run indexing again.'); + let removed = false; + if (fs.existsSync(lockPath)) { + fs.unlinkSync(lockPath); + removed = true; + } + const { clearStaleDaemonArtifacts } = await import('../mcp/daemon-registry'); + removed = await clearStaleDaemonArtifacts(projectPath) || removed; + if (removed) success('Removed stale lock artifacts. You can now run indexing again.'); + else info(`No stale lock files found ${getGlyphs().dash} nothing to do`); } catch (err) { error(`Failed to remove lock: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); diff --git a/src/db/index.ts b/src/db/index.ts index 4d52b0c6c..f01d195d1 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -145,6 +145,7 @@ export class DatabaseConnection { // beginBulkNodeLoad and endBulkNodeLoad): the FTS triggers are missing and // nodes_fts is stale. Rebuild + recreate so search stays in sync. conn.healBulkNodeLoad(); + conn.healBulkSecondaryIndexes(); // Self-heal a killed session's leftover oversized WAL (#1431) — one // statSync when healthy, off-thread checkpoint+truncate when not. @@ -363,6 +364,28 @@ export class DatabaseConnection { this.endBulkNodeLoad(); } + /** Recreate every secondary index a killed bulk parse/ref/edge window may leave dropped. */ + private healBulkSecondaryIndexes(): void { + const names = [...new Set([ + ...DatabaseConnection.BULK_PARSE_INDEX_NAMES, + ...DatabaseConnection.BULK_REF_INDEX_NAMES, + ...DatabaseConnection.BULK_EDGE_INDEX_NAMES, + ])]; + const placeholders = names.map(() => '?').join(','); + const row = this.db + .prepare(`SELECT count(*) AS c FROM sqlite_master WHERE type = 'index' AND name IN (${placeholders})`) + .get(...names) as { c: number } | undefined; + if ((row?.c ?? 0) >= names.length) return; + + const schemaPath = path.join(__dirname, 'schema.sql'); + const schema = fs.readFileSync(schemaPath, 'utf-8'); + for (const idx of names) { + const m = schema.match(new RegExp(`CREATE INDEX IF NOT EXISTS ${idx}\\b[^;]*;`)); + if (!m) throw new Error(`schema.sql: index ${idx} not found for crash recovery`); + this.db.exec(m[0]); + } + } + /** * Recreate the FTS sync triggers from schema.sql — extracted from the file * rather than duplicated here so the DDL cannot drift from the schema. diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 3606d2591..2b61636b6 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -1714,7 +1714,7 @@ export class ExtractionOrchestrator { const inFlight = new Set>(); const completed = new Map(); + | { ok: false; filePath: string; content: string; stats: fs.Stats; err: unknown }>(); let nextSeq = 0; // file-order sequence assigned at dispatch let nextToStore = 0; // cursor: next sequence to commit let aborted = false; @@ -1742,27 +1742,25 @@ export class ExtractionOrchestrator { // Store: on the writer thread when active (fresh DB — bundles applied // in the same file order this chain dispatches them), else on the main // thread (SQLite connections are per-thread). - if (nodeCount > 0 || result.errors.length === 0) { - const language = detectLanguage(filePath, content, overrides); - if (storeWriter) { - if (result.kernelBuffers) { - // Buffers go to the writer as-is; the worker decodes + finalizes. - // The main thread's only per-file work stays O(1) + the content hash. - storeWriter.send({ - kernel: true, - filePath, - language, - buffers: result.kernelBuffers, - file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors), - }); - } else { - storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result)); - } - await storeWriter.waitBelow(STORE_WRITER_WINDOW); + const language = detectLanguage(filePath, content, overrides); + if (storeWriter) { + if (result.kernelBuffers) { + // Buffers go to the writer as-is; the worker decodes + finalizes. + // The main thread's only per-file work stays O(1) + the content hash. + storeWriter.send({ + kernel: true, + filePath, + language, + buffers: result.kernelBuffers, + file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors), + }); } else { - const materialized = materializeKernelResult(result, filePath, language); - await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield); + storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result)); } + await storeWriter.waitBelow(STORE_WRITER_WINDOW); + } else { + const materialized = materializeKernelResult(result, filePath, language); + await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield); } if (result.errors.length > 0) { @@ -1793,16 +1791,19 @@ export class ExtractionOrchestrator { onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath }); }; - const recordParseFailure = (filePath: string, err: unknown): void => { - processed++; - filesErrored++; - errors.push({ - message: err instanceof Error ? err.message : String(err), - filePath, - severity: 'error', - code: 'parse_error', + const recordParseFailure = async (filePath: string, content: string, stats: fs.Stats, err: unknown): Promise => { + await storeResult(filePath, content, stats, { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: [{ + message: err instanceof Error ? err.message : String(err), + filePath, + severity: 'error', + code: 'parse_error', + }], + durationMs: 0, }); - onProgress?.({ phase: 'parsing', current: processed, total }); }; // Commit buffered parses to the DB in file order, advancing the cursor over @@ -1825,7 +1826,7 @@ export class ExtractionOrchestrator { completed.delete(nextToStore); nextToStore++; if (item.ok) await storeResult(item.filePath, item.content, item.stats, item.result); - else recordParseFailure(item.filePath, item.err); + else await recordParseFailure(item.filePath, item.content, item.stats, item.err); } } catch (err) { flushError = err; @@ -1844,7 +1845,7 @@ export class ExtractionOrchestrator { const result = await parseFile(filePath, content); completed.set(seq, { ok: true, filePath, content, stats, result }); } catch (parseErr) { - completed.set(seq, { ok: false, filePath, err: parseErr }); + completed.set(seq, { ok: false, filePath, content, stats, err: parseErr }); } flushOrdered(); })(); @@ -1915,15 +1916,18 @@ export class ExtractionOrchestrator { // useful symbols. The single-file extractFile path already enforces // this; the bulk path used to silently skip the check. if (stats.size > MAX_FILE_SIZE) { - processed++; - filesSkipped++; - errors.push({ - message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`, - filePath, - severity: 'warning', - code: 'size_exceeded', + await storeResult(filePath, content, stats, { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: [{ + message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`, + filePath, + severity: 'warning', + code: 'size_exceeded', + }], + durationMs: 0, }); - onProgress?.({ phase: 'parsing', current: processed, total }); continue; } @@ -2242,9 +2246,11 @@ export class ExtractionOrchestrator { }; } + const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir)); + // Check file size if (stats.size > MAX_FILE_SIZE) { - return { + const result: ExtractionResult = { nodes: [], edges: [], unresolvedReferences: [], @@ -2258,10 +2264,11 @@ export class ExtractionOrchestrator { ], durationMs: 0, }; + await this.storeExtractionResult(relativePath, content, language, stats, result, createYielder()); + return result; } // Detect language (honoring the project's codegraph.json extension overrides) - const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir)); if (!isLanguageSupported(language)) { return { nodes: [], @@ -2279,9 +2286,7 @@ export class ExtractionOrchestrator { const result = extractFromSource(relativePath, content, language, frameworkNames); // Store in database - if (result.nodes.length > 0 || result.errors.length === 0) { - await this.storeExtractionResult(relativePath, content, language, stats, result, createYielder()); - } + await this.storeExtractionResult(relativePath, content, language, stats, result, createYielder()); return result; } @@ -2303,7 +2308,15 @@ export class ExtractionOrchestrator { */ private healZeroNodeRows(): void { for (const f of this.queries.getAllFiles()) { - if (f.nodeCount === 0 && !isFileLevelOnlyLanguage(f.language)) { + // A zero-node row WITH recorded errors is a deliberate skip marker + // (#1557: oversized / repeatedly-unparseable files are persisted with + // their reason so syncs stop retrying them) — leave those alone. The + // #1541 wipe rows are the error-FREE zero-node rows. + if ( + f.nodeCount === 0 && + !isFileLevelOnlyLanguage(f.language) && + (f.errors === undefined || f.errors.length === 0) + ) { this.queries.deleteFile(f.path); } } @@ -2332,10 +2345,20 @@ export class ExtractionOrchestrator { const STORE_CHUNK = 2000; const contentHash = hashContent(content); - // Check if file already exists and hasn't changed + // Check if file already exists and hasn't changed. A skip/failure MARKER + // row (zero nodes + recorded errors, #1557) never blocks a store carrying + // real content: markers are written BEFORE the retry pass under the same + // content hash, so treating them as "no changes" would silently discard a + // successful retry's symbols — a permanent empty file presented as + // recovered (the #1541 wipe, reintroduced through the marker path). const existingFile = this.queries.getFileByPath(filePath); if (existingFile && existingFile.contentHash === contentHash) { - return; // No changes + const existingIsMarker = + existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0; + const incomingHasContent = result.nodes.length > 0; + if (!existingIsMarker || !incomingHasContent) { + return; // No changes + } } // Re-decided on every re-index of a changed file, so a banner added (or diff --git a/src/extraction/parse-pool.ts b/src/extraction/parse-pool.ts index 26f8ca055..c0cacd216 100644 --- a/src/extraction/parse-pool.ts +++ b/src/extraction/parse-pool.ts @@ -61,6 +61,8 @@ const MAX_PARSE_POOL_SIZE = 16; const DEFAULT_RECYCLE_INTERVAL = 250; /** Base per-parse timeout; scaled up for large files by the caller's formula. */ const DEFAULT_PARSE_TIMEOUT_MS = 10_000; +/** Keep the default large-file budget bounded; the hard-kill window is 3× this. */ +const MAX_SCALED_PARSE_TIMEOUT_MS = 20_000; /** * A worker is only killed once a parse has gone this many × its budget with no * result. The base timer firing is NOT proof the parse is still running: after @@ -109,6 +111,17 @@ export function resolveParseTimeoutMs(envVal: string | undefined): number { return DEFAULT_PARSE_TIMEOUT_MS; } +/** + * Per-file soft timeout. Size scaling helps legitimate large sources, but an + * uncapped linear budget gave data-only headers near the 1 MiB file limit a + * 4.5–5 minute hard-kill window (#1555). Explicit larger base overrides remain + * respected for slow storage. + */ +export function resolveParseBudgetMs(baseMs: number, contentLength: number): number { + const scaled = baseMs + Math.floor(contentLength / 100_000) * 10_000; + return Math.min(scaled, Math.max(baseMs, MAX_SCALED_PARSE_TIMEOUT_MS)); +} + export function resolveParsePoolSize(envVal: string | undefined, cpuCount: number): number { if (envVal !== undefined && envVal !== '') { const n = Number(envVal); @@ -344,7 +357,7 @@ export class ParseWorkerPool { this.parseCounts.set(w, (this.parseCounts.get(w) ?? 0) + 1); // Scale the timeout for large files: base + 10s per 100KB (matches the // original single-worker formula so pathological-file behaviour is unchanged). - const timeoutMs = this.parseTimeoutMs + Math.floor(job.task.content.length / 100_000) * 10_000; + const timeoutMs = resolveParseBudgetMs(this.parseTimeoutMs, job.task.content.length); job.budgetMs = timeoutMs; job.timer = setTimeout(() => this.onTimeout(w, job, timeoutMs), timeoutMs); job.timer.unref?.(); diff --git a/src/index.ts b/src/index.ts index 15eaf7a7a..2942575b5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -976,6 +976,14 @@ export class CodeGraph { } } catch { /* vocab is advisory — never fail a sync over it */ } + // A killed full index leaves this marker at `indexing`. Sync repairs + // missing files, pending refs, and (on open) dropped indexes, so a + // successful recovery must also close the metadata state (#1556). + const fullReconcile = !options.paths || options.paths.length === 0; + if (fullReconcile && this.getIndexState() === 'indexing') { + try { this.queries.setMetadata('index_state', 'complete'); } catch { /* advisory */ } + } + return result; } finally { // Mirror indexAll's teardown: stop the valve, then restore the diff --git a/src/mcp/daemon-manager.ts b/src/mcp/daemon-manager.ts index 47a61e077..0c1a991a1 100644 --- a/src/mcp/daemon-manager.ts +++ b/src/mcp/daemon-manager.ts @@ -61,7 +61,7 @@ export function buildPickItems(daemons: DaemonRecord[], cwdRoot: string | null, } export interface PickerDeps { - list: () => DaemonRecord[]; + list: () => DaemonRecord[] | Promise; stop: (root: string) => Promise; stopAll: () => Promise; /** Realpath'd root of the current project's daemon, or null. */ @@ -82,7 +82,7 @@ export interface PickerDeps { */ export async function runDaemonPicker(deps: PickerDeps): Promise { for (;;) { - const daemons = deps.list(); + const daemons = await deps.list(); if (daemons.length === 0) { deps.done('All daemons stopped.'); return; diff --git a/src/mcp/daemon-paths.ts b/src/mcp/daemon-paths.ts index 13f19045f..c860ee76c 100644 --- a/src/mcp/daemon-paths.ts +++ b/src/mcp/daemon-paths.ts @@ -29,6 +29,7 @@ */ import * as crypto from 'crypto'; +import * as net from 'net'; import * as os from 'os'; import * as path from 'path'; import { getCodeGraphDir } from '../directory'; @@ -101,6 +102,55 @@ export interface DaemonLockInfo { startedAt: number; } +/** + * Verify that the process named by a lockfile is the CodeGraph daemon serving + * its socket. A bare PID liveness probe is insufficient because OSes reuse PIDs + * after an OOM/SIGKILL (#1553). + */ +export function probeDaemonIdentity(info: DaemonLockInfo, timeoutMs = 1_000): Promise { + if (!Number.isInteger(info.pid) || info.pid <= 0 || !info.socketPath) return Promise.resolve(false); + return new Promise((resolve) => { + let socket: net.Socket; + let buffer = ''; + let done = false; + const finish = (ok: boolean) => { + if (done) return; + done = true; + clearTimeout(timer); + socket.destroy(); + resolve(ok); + }; + const timer = setTimeout(() => finish(false), timeoutMs); + timer.unref?.(); + try { + socket = net.createConnection(info.socketPath); + } catch { + clearTimeout(timer); + resolve(false); + return; + } + socket.setEncoding('utf8'); + socket.on('data', (chunk) => { + buffer += String(chunk); + if (buffer.length > 4096) return finish(false); + const newline = buffer.indexOf('\n'); + if (newline < 0) return; + try { + const hello = JSON.parse(buffer.slice(0, newline)) as Record; + finish( + hello.protocol === 1 && + hello.pid === info.pid && + (info.version === 'unknown' || hello.codegraph === info.version) + ); + } catch { + finish(false); + } + }); + socket.on('error', () => finish(false)); + socket.on('close', () => finish(false)); + }); +} + /** * Serialize a {@link DaemonLockInfo} for writing to the pidfile. JSON for * human readability — operators occasionally `cat` this when debugging. diff --git a/src/mcp/daemon-registry.ts b/src/mcp/daemon-registry.ts index e1885361c..f731563cf 100644 --- a/src/mcp/daemon-registry.ts +++ b/src/mcp/daemon-registry.ts @@ -22,7 +22,13 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import * as crypto from 'crypto'; -import { getDaemonPidPath, getDaemonSocketCandidates, decodeLockInfo } from './daemon-paths'; +import { + getDaemonPidPath, + getDaemonSocketCandidates, + decodeLockInfo, + probeDaemonIdentity, + type DaemonLockInfo, +} from './daemon-paths'; export interface DaemonRecord { /** Realpath'd project root the daemon serves. */ @@ -114,6 +120,26 @@ export function listDaemons(opts: { prune?: boolean } = {}): DaemonRecord[] { return live.sort((a, b) => b.startedAt - a.startedAt); } +/** + * Registry entries whose socket hello proves the recorded process is the + * daemon. Used by every user-facing list/stop-all path so a reused PID cannot + * appear as a phantom running daemon (#1553). + */ +export async function listVerifiedDaemons(opts: { prune?: boolean } = {}): Promise { + const prune = opts.prune ?? true; + const candidates = listDaemons({ prune }); + const checks = await Promise.all(candidates.map(async (rec) => ({ + rec, + verified: await probeDaemonIdentity(rec), + }))); + const verified: DaemonRecord[] = []; + for (const check of checks) { + if (check.verified) verified.push(check.rec); + else if (prune) deregisterDaemon(check.rec.root); + } + return verified; +} + /** Remove a stopped daemon's leftover lockfile + socket + registry record. */ function cleanupDaemonArtifacts(root: string): void { try { fs.unlinkSync(getDaemonPidPath(root)); } catch { /* gone */ } @@ -128,6 +154,20 @@ function cleanupDaemonArtifacts(root: string): void { deregisterDaemon(root); } +/** Remove daemon artifacts only when no matching daemon answers the socket hello. */ +export async function clearStaleDaemonArtifacts(root: string): Promise { + const pidPath = getDaemonPidPath(root); + const hadArtifacts = fs.existsSync(pidPath) || ( + process.platform !== 'win32' && getDaemonSocketCandidates(root).some((p) => fs.existsSync(p)) + ); + if (!hadArtifacts) return false; + let info: DaemonLockInfo | null = null; + try { info = decodeLockInfo(fs.readFileSync(pidPath, 'utf8')); } catch { /* missing/corrupt */ } + if (info && isProcessAlive(info.pid) && await probeDaemonIdentity(info)) return false; + cleanupDaemonArtifacts(root); + return true; +} + const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); async function waitForDeath(pid: number, timeoutMs: number): Promise { @@ -154,9 +194,10 @@ export interface StopResult { */ export async function stopDaemonAt(root: string): Promise { let pid: number | null = null; + let identity: DaemonLockInfo | null = null; try { - const info = decodeLockInfo(fs.readFileSync(getDaemonPidPath(root), 'utf8')); - pid = info?.pid ?? null; + identity = decodeLockInfo(fs.readFileSync(getDaemonPidPath(root), 'utf8')); + pid = identity?.pid ?? null; } catch { /* no lockfile */ } @@ -165,6 +206,7 @@ export async function stopDaemonAt(root: string): Promise { (r) => path.resolve(r.root) === path.resolve(root) ); pid = rec?.pid ?? null; + if (rec) identity = rec; } if (pid == null) { @@ -175,6 +217,12 @@ export async function stopDaemonAt(root: string): Promise { cleanupDaemonArtifacts(root); return { root, pid, outcome: 'not-running' }; } + // Never signal a process merely because it reused a stale daemon PID. The + // daemon's immediate hello is the process-identity proof (#1553). + if (!identity || !await probeDaemonIdentity(identity)) { + cleanupDaemonArtifacts(root); + return { root, pid, outcome: 'not-running' }; + } // POSIX: SIGTERM runs the daemon's graceful shutdown. Windows: TerminateProcess // (no graceful path), so we always sweep artifacts ourselves below. @@ -192,7 +240,7 @@ export async function stopDaemonAt(root: string): Promise { /** Stop every registered, live daemon. */ export async function stopAllDaemons(): Promise { const results: StopResult[] = []; - for (const rec of listDaemons()) { + for (const rec of await listVerifiedDaemons()) { results.push(await stopDaemonAt(rec.root)); } return results; diff --git a/src/mcp/daemon.ts b/src/mcp/daemon.ts index b1d45328b..500c48a8c 100644 --- a/src/mcp/daemon.ts +++ b/src/mcp/daemon.ts @@ -629,25 +629,31 @@ export function acquireLockViaExclusiveOpen(pidPath: string, info: DaemonLockInf } /** - * Remove a stale pidfile, but only if it still names a dead process. Re-reads - * the file immediately before unlinking so we never delete a lock that a live - * daemon (re)acquired in the meantime. + * Remove a stale pidfile. Re-reads the file immediately before unlinking so a + * different daemon that acquired the lock in the meantime is never disturbed. * * must-fix 1 (issue #411 review): the original unconditionally `unlink`'d, * which let a racing candidate delete a healthy daemon's lock. Passing * `expectedDeadPid` (the pid the caller believed was dead) makes the clear a - * compare-and-delete: bail if the file now holds a different pid, or any live - * pid. Returns true when the stale lock is gone (or was already gone). + * compare-and-delete: bail if the file now holds a different pid. By default a + * live pid is also preserved; `allowLivePid` is reserved for callers that have + * already disproved daemon identity with the socket hello (#1553). Returns true + * when the stale lock is gone (or was already gone). */ -export function clearStaleDaemonLock(pidPath: string, expectedDeadPid?: number): boolean { +export function clearStaleDaemonLock( + pidPath: string, + expectedDeadPid?: number, + opts: { allowLivePid?: boolean } = {} +): boolean { try { const raw = fs.readFileSync(pidPath, 'utf8'); const info = decodeLockInfo(raw); if (info) { // A different pid took over since we read it — not ours to clear. if (expectedDeadPid !== undefined && info.pid !== expectedDeadPid) return false; - // Holder is actually alive — never clear a live daemon's lock. - if (info.pid > 0 && isProcessAlive(info.pid)) return false; + // PID liveness is normally sufficient. The takeover caller may override + // it only after a failed identity handshake proves PID reuse. + if (!opts.allowLivePid && info.pid > 0 && isProcessAlive(info.pid)) return false; } fs.unlinkSync(pidPath); return true; diff --git a/src/mcp/index.ts b/src/mcp/index.ts index c7c59f622..971121054 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -48,7 +48,7 @@ import { tryAcquireDaemonLock, } from './daemon'; import { connectWithHello, runLocalHandshakeProxy } from './proxy'; -import { getDaemonSocketCandidates } from './daemon-paths'; +import { getDaemonSocketCandidates, probeDaemonIdentity } from './daemon-paths'; import { getTelemetry } from '../telemetry'; import { checkForUpdateInBackground } from '../upgrade/update-check'; import { EARLY_PPID } from './early-ppid'; @@ -423,15 +423,22 @@ export class MCPServer { // binding) — we're redundant; exit cleanly so the launcher proxies to it. const existing = lock.existing; if (existing && existing.pid > 0 && isProcessAlive(existing.pid)) { - process.stderr.write( - `[CodeGraph daemon] Another daemon (pid ${existing.pid}) already holds the lock; exiting.\n` - ); - process.exit(0); + // Give a newly-elected daemon time to bind, then require its socket hello + // to match the lock PID/version. PID existence alone accepts an unrelated + // process after OS PID reuse and permanently wedges startup (#1553). + const age = Date.now() - existing.startedAt; + const stillStarting = existing.startedAt > 0 && age >= 0 && age < 10_000; + if (stillStarting || await probeDaemonIdentity(existing)) { + process.stderr.write( + `[CodeGraph daemon] Another daemon (pid ${existing.pid}) already holds the lock; exiting.\n` + ); + process.exit(0); + } } // Holder is dead (or the record is unreadable) — clear it (pid-verified, // so we never delete a live daemon's lock) and retry the acquire. - clearStaleDaemonLock(lock.pidPath, existing?.pid); + clearStaleDaemonLock(lock.pidPath, existing?.pid, { allowLivePid: true }); await sleep(TAKEOVER_RETRY_DELAY_MS); } diff --git a/src/resolution/c-fnptr-synthesizer.ts b/src/resolution/c-fnptr-synthesizer.ts index 1ab809918..568b782c9 100644 --- a/src/resolution/c-fnptr-synthesizer.ts +++ b/src/resolution/c-fnptr-synthesizer.ts @@ -1209,7 +1209,7 @@ export async function cFnPointerDispatchEdges( // ---- receiver-type resolution within a function's source ---- // `(?:struct )?TYPE [*]recv` declared in the params or body → TYPE (if a known // fn-pointer-bearing struct). - const recvReCache = new Map(); + const recvReCache = new LRUCache(4096); const recvTypeIn = (fnSrc: string, recv: string): string | null => { let re = recvReCache.get(recv); if (!re) { @@ -1228,7 +1228,7 @@ export async function cFnPointerDispatchEdges( // structs (the base of a chained receiver needn't carry a fn pointer itself). // Falls back to a file-scope table variable (`cmdnames` in `cmdnames[i].fn()`). const escapeRe = (x: string): string => x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const varReCache = new Map(); + const varReCache = new LRUCache(4096); const varTypeIn = (fnSrc: string, v: string): string | null => { let re = varReCache.get(v); if (!re) { diff --git a/src/resolution/callback-synthesizer.ts b/src/resolution/callback-synthesizer.ts index dc8333149..60b389937 100644 --- a/src/resolution/callback-synthesizer.ts +++ b/src/resolution/callback-synthesizer.ts @@ -1241,7 +1241,12 @@ async function reactJsxChildEdges(ctx: ResolutionContext, onYield: MaybeYield): if ((++scanned & 255) === 0) await onYield(); // #1091: yield mid-scan on huge graphs const content = ctx.readFile(file); if (!content || (!content.includes(''))) continue; // JSX-file gate - const parents = ctx.getNodesInFile(file).filter((n) => PARENT_KINDS.has(n.kind)); + // File-level language gate, not merely a project-level one: mixed C/JS + // monorepos must not interpret `""` inside C as JSX (#1560). + const parents = ctx.getNodesInFile(file).filter( + (n) => PARENT_KINDS.has(n.kind) && JS_FAMILY.includes(n.language) + ); + if (parents.length === 0) continue; for (const parent of parents) { const src = sliceLines(content, parent.startLine, parent.endLine); if (!src || (!src.includes(''))) continue; @@ -3533,7 +3538,7 @@ export const SYNTH_PASSES: SynthPassDef[] = [ { name: 'closureCollEdges', gate: ALWAYS, run: (q, c, y) => closureCollectionEdges(q, c, y) }, { name: 'emitterEdges', gate: ALWAYS, run: (_q, c, y) => eventEmitterEdges(c, y) }, { name: 'renderEdges', gate: ALWAYS, run: (q, c, y) => reactRenderEdges(q, c, y) }, - { name: 'jsxEdges', gate: ALWAYS, run: (_q, c, y) => reactJsxChildEdges(c, y) }, + { name: 'jsxEdges', gate: (has) => has(...JS_FAMILY), run: (_q, c, y) => reactJsxChildEdges(c, y) }, { name: 'vueEdges', gate: (has) => has('vue'), run: (_q, c, y) => vueTemplateEdges(c, y) }, { name: 'svelteKitEdges', gate: (has) => has('svelte'), run: (_q, c, y) => svelteKitLoadEdges(c, y) }, { name: 'pascalEdges', gate: ALWAYS, run: (_q, c, y) => pascalFormEdges(c, y) }, From ccb02952598732e8ab6bc1946e2f271836c0a58f Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Sat, 22 Aug 2026 09:05:30 -0700 Subject: [PATCH 06/27] fix(explore): reliably pin extension-less kebab-case file basenames in queries Previously, naming a kebab-case file without its extension (e.g., `background-image-table` vs. `background-image-table.tsx`) in a `codegraph_explore` query would shred the name into fragments (`background`, `image`, `table`), admitting irrelevant sibling files and crowding out the intended target. This change introduces a new resolution pass in `extractQueryPaths` specifically for extension-less kebab basenames. Queries now accurately identify and pin these files. Unresolved hyphenated prose (e.g., `cross-call`) is left in the query for FTS without being flagged as an unknown path. Resolution prioritizes explicit slashed/dotted paths and respects an ambiguity budget for common stems to prevent over-pinning. --- CHANGELOG.md | 1 + __tests__/explore-path-pinning.test.ts | 17 +++ .../src/lib/background-image-table.ts | 21 ++++ .../src/lib/background-store.ts | 11 ++ __tests__/query-paths.test.ts | 109 ++++++++++++++++++ src/search/query-paths.ts | 90 +++++++++++++-- 6 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 __tests__/fixtures/explore-path-pinning/src/lib/background-image-table.ts create mode 100644 __tests__/fixtures/explore-path-pinning/src/lib/background-store.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f44be8edd..aebf1653d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes - Naming a file by its path in a `codegraph_explore` query now works reliably: the path is resolved against the index and that file is guaranteed a place at the top of the answer. Previously the path was broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out of the answer entirely. A path that doesn't match any indexed file is now called out instead of silently ignored. +- Naming a kebab-case file **without its extension** in a `codegraph_explore` query — `background-image-table` rather than `background-image-table.tsx`, the way import paths and prose spell it — now returns that exact file too. Previously the name was split at the hyphens, and in a kebab-cased frontend those pieces (`background`, `image`, `table`) are among the most common words in the codebase, so look-alike sibling files filled the answer while the named file never appeared. Hyphenated words that don't name an indexed file, like "cross-call" or "non-blocking", are left alone. - Plainly-worded `codegraph_explore` questions now find camelCase code: a query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names. - Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked. - C, C++, Objective-C and Rust unions are now indexed as first-class `union` nodes. A `union` declaration previously produced no symbol at all, so it never appeared in search or `codegraph_explore`, and anything attached to it disappeared with it — in Rust, every `impl SomeTrait for MyUnion` lost its edge, the methods from that impl were left pointing at a type the graph did not contain, and asking which types implement a trait quietly skipped the union ones. A union-shaped dispatch table in C now resolves its function pointers like a struct-shaped one. A `typedef union { … } Name;` in C keeps the typedef's name and is no longer mistaken for a plain type alias. Thanks @ctype-lab. Re-index after upgrading to pick up unions in existing projects. (#1515) diff --git a/__tests__/explore-path-pinning.test.ts b/__tests__/explore-path-pinning.test.ts index 8ec15e1f2..b79ba7d12 100644 --- a/__tests__/explore-path-pinning.test.ts +++ b/__tests__/explore-path-pinning.test.ts @@ -92,6 +92,23 @@ describe('path pinning (fix 1)', () => { }); }); +describe('extension-less kebab basenames (the amnisphere gap)', () => { + const KEBAB_TARGET = 'src/lib/background-image-table.ts'; + + it('a bare kebab basename — no slash, no extension — pins and renders its file', async () => { + // Pre-fix this query never opened the path gate; FTS shredded the token + // into `background`/`image`/`table` and served the fragment decoy instead. + const out = await explore('background-image-table Source column'); + expect(hasSection(out, KEBAB_TARGET)).toBe(true); + expect(out).toContain('pinned from the query'); + }); + + it('kebab prose that names no file is not reported as an unresolved path', async () => { + const out = await explore('how does cross-call dedup interact with feed scroll pinning'); + expect(out).not.toContain('No indexed file uniquely matches'); + }); +}); + describe('segment supplement + variable seeding (fixes 2–3)', () => { it('word-level scroll terms reach the camelCase scroll code without a path', async () => { const out = await explore('feed auto-scroll to bottom pinning behavior'); diff --git a/__tests__/fixtures/explore-path-pinning/src/lib/background-image-table.ts b/__tests__/fixtures/explore-path-pinning/src/lib/background-image-table.ts new file mode 100644 index 000000000..e77d83334 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/lib/background-image-table.ts @@ -0,0 +1,21 @@ +/** Table of background images for a training set — source-column rendering. */ + +export interface BackgroundImageRow { + id: string; + sourceUrl: string; + label: string; +} + +let tableRows: BackgroundImageRow[] = []; + +export function loadTableRows(rows: BackgroundImageRow[]): void { + tableRows = rows; +} + +export function renderSourceColumn(row: BackgroundImageRow): string { + return `${row.label}: ${row.sourceUrl}`; +} + +export function sortRowsBySource(): BackgroundImageRow[] { + return [...tableRows].sort((a, b) => a.sourceUrl.localeCompare(b.sourceUrl)); +} diff --git a/__tests__/fixtures/explore-path-pinning/src/lib/background-store.ts b/__tests__/fixtures/explore-path-pinning/src/lib/background-store.ts new file mode 100644 index 000000000..541986e26 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/lib/background-store.ts @@ -0,0 +1,11 @@ +/** Uploaded-background registry — shares the `background` fragment with the table file. */ + +let backgrounds: string[] = []; + +export function addBackground(url: string): void { + backgrounds.push(url); +} + +export function listBackgrounds(): string[] { + return [...backgrounds]; +} diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts index 13a47470e..f1e3c0e2e 100644 --- a/__tests__/query-paths.test.ts +++ b/__tests__/query-paths.test.ts @@ -22,6 +22,21 @@ const INDEX = [ 'src/lib/task-runner-manager.ts', 'src/lib/stores/sqlite-store.ts', 'src/lib/stores/postgresql-store.ts', + // Kebab-case frontend shapes (the amnisphere extension-less-basename bug): + 'src/components/training-set-page/training-set-page.tsx', + 'src/components/training-set-page/training-set-page-background-images.tsx', + 'src/components/training-set-page/training-set-page.module.scss', + 'src/components/training-set-page/background-image-table.tsx', + 'src/components/modal/add-to-training-set/add-to-training-set.tsx', + 'src/pages/library-page-layout.tsx', + 'src/api/job-manager/backgrounds.ts', + 'src/x/generic-modal.tsx', + 'src/y/generic-modal.tsx', + 'scripts/pre-commit', + 'src/a/user-profile.tsx', + 'src/b/user-profile.tsx', + 'src/c/user-profile.tsx', + 'src/d/user-profile.tsx', ]; describe('queryMightContainPaths — the cheap pre-gate', () => { @@ -35,6 +50,18 @@ describe('queryMightContainPaths — the cheap pre-gate', () => { // `.isPackaged` is 10 chars — past the 8-char extension cap. expect(queryMightContainPaths('what reads app.isPackaged here')).toBe(false); }); + + it('fires on extension-less kebab basenames — with or without wrapping', () => { + expect(queryMightContainPaths('background-image-table Source column')).toBe(true); + expect(queryMightContainPaths('the `library-page-layout` wrapper')).toBe(true); + expect(queryMightContainPaths('usage, add-to-training-set.')).toBe(true); + }); + + it('stays quiet on flags, snake_case, and snake-with-a-dash hybrids', () => { + expect(queryMightContainPaths('run it with --no-cache maybe')).toBe(false); + expect(queryMightContainPaths('where is background_image_table used')).toBe(false); + expect(queryMightContainPaths('the foo_bar-baz helper')).toBe(false); + }); }); describe('extractQueryPaths — resolution and stripping', () => { @@ -127,3 +154,85 @@ describe('extractQueryPaths — resolution and stripping', () => { expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); }); }); + +describe('extractQueryPaths — extension-less kebab basenames', () => { + it('pins the file a bare kebab basename names and consumes the token', () => { + const out = extractQueryPaths('background-image-table Source column', INDEX); + expect(out.pinnedFiles) + .toEqual(['src/components/training-set-page/background-image-table.tsx']); + expect(out.strippedQuery).toBe('Source column'); + expect(out.unresolvedPathSpans).toEqual([]); + }); + + it('resolves with no slash or extension anywhere in the query (session-4 shape)', () => { + const out = extractQueryPaths( + 'TrainingSetPage train modal library-page-layout AddToTrainingSetModal usage', INDEX, + ); + expect(out.pinnedFiles).toEqual(['src/pages/library-page-layout.tsx']); + // Identifier-shaped tokens stay for the named-symbol seeder. + expect(out.strippedQuery).toBe('TrainingSetPage train modal AddToTrainingSetModal usage'); + }); + + it('pins every named file in a mixed dotted + kebab query (session-1 shape)', () => { + const out = extractQueryPaths( + 'add-to-training-set training-set-page-background-images backgrounds.ts background-image-table Source column', + INDEX, + ); + expect(out.pinnedFiles).toEqual([ + // The dotted pass runs first, so the explicit basename pins ahead of the kebabs. + 'src/api/job-manager/backgrounds.ts', + 'src/components/modal/add-to-training-set/add-to-training-set.tsx', + 'src/components/training-set-page/training-set-page-background-images.tsx', + 'src/components/training-set-page/background-image-table.tsx', + ]); + expect(out.strippedQuery).toBe('Source column'); + }); + + it('leaves kebab prose that names no indexed file untouched — and unreported', () => { + const q = 'how does cross-call dedup make explore non-blocking'; + const out = extractQueryPaths(q, INDEX); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + }); + + it('leaves a stem shared by too many files alone — one hot name must not pin half the repo', () => { + const q = 'refactor the user-profile rendering'; + const out = extractQueryPaths(q, INDEX); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + }); + + it('pins all files sharing a stem when within the ambiguity budget', () => { + const out = extractQueryPaths('generic-modal close behavior', INDEX); + expect(out.pinnedFiles).toEqual(['src/x/generic-modal.tsx', 'src/y/generic-modal.tsx']); + }); + + it('matches case-insensitively and through wrapping punctuation', () => { + expect(extractQueryPaths('see `Background-Image-Table`.', INDEX).pinnedFiles) + .toEqual(['src/components/training-set-page/background-image-table.tsx']); + }); + + it('stems drop only the last extension — a kebab token cannot pin a .module.scss sibling', () => { + const out = extractQueryPaths('training-set-page props flow', INDEX); + expect(out.pinnedFiles) + .toEqual(['src/components/training-set-page/training-set-page.tsx']); + }); + + it('pins an extension-less indexed file by its exact name', () => { + expect(extractQueryPaths('what does the pre-commit hook run', INDEX).pinnedFiles) + .toEqual(['scripts/pre-commit']); + }); + + it('skips tokens the dotted pass consumed and dedupes a file named both ways', () => { + const out = extractQueryPaths('src/lib/chat-manager.ts vs chat-manager internals', INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + expect(out.strippedQuery).toBe('vs internals'); + }); + + it('explicit paths win the shared maxPins budget over kebab tokens', () => { + const out = extractQueryPaths( + 'background-image-table then src/lib/chat-manager.ts', INDEX, { maxPins: 1 }, + ); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + // The kebab token was not consumed once the budget was spent — it stays for FTS. + expect(out.strippedQuery).toBe('background-image-table then'); + }); +}); diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts index ce186c8a2..c91272ddb 100644 --- a/src/search/query-paths.ts +++ b/src/search/query-paths.ts @@ -15,10 +15,13 @@ * sibling `+page.svelte` in the repo, which ate the output envelope and * truncated the files the agent actually asked for. * - * `extractQueryPaths` finds path-like spans, resolves them against the - * INDEXED file list (resolution IS the detector — `and/or`, `gen_server:call/2` - * and other slash-bearing non-paths match nothing and are left alone), and - * returns the matches as pinned files plus the query with those spans removed. + * `extractQueryPaths` finds path-like spans — slashed paths, dotted basenames, + * and extension-less kebab basenames (`background-image-table`, the spelling + * import paths and prose actually use) — resolves them against the INDEXED + * file list (resolution IS the detector — `and/or`, `gen_server:call/2`, + * `non-blocking` and other path-shaped non-paths match nothing and are left + * alone), and returns the matches as pinned files plus the query with those + * spans removed. * Callers treat pinned files as first-class: guaranteed admission, top rank, * funded first. Pure string work — no DB, no fs — so it is trivially testable * and safe inside the query-pool workers. @@ -40,12 +43,18 @@ export interface QueryPathExtraction { /** * Cheap pre-gate so callers only fetch the indexed file list when the query - * could possibly contain a path: a slash, or a dot-extension-shaped tail - * (`chat-manager.ts`). Extensions cap at 8 chars, which keeps `Class.method` - * spans (`app.isPackaged`) from qualifying. + * could possibly contain a path: a slash, a dot-extension-shaped tail + * (`chat-manager.ts`), or a hyphen-joined word (`background-image-table` — + * kebab files are named WITHOUT their extension more often than with, so the + * shape must open the gate on its own). Extensions cap at 8 chars, which + * keeps `Class.method` spans (`app.isPackaged`) from qualifying; the kebab + * alternative requires clean non-word boundaries, which keeps `--flags` and + * snake_case-with-a-dash hybrids from firing it. */ export function queryMightContainPaths(query: string): boolean { - return /[/\\]/.test(query) || /\.[A-Za-z][A-Za-z0-9]{0,7}(?=[\s,;:)\]'"`]|$)/.test(query); + return /[/\\]/.test(query) + || /\.[A-Za-z][A-Za-z0-9]{0,7}(?=[\s,;:)\]'"`]|$)/.test(query) + || /(?:^|[^-\w])[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+(?=[^-\w]|$)/.test(query); } /** @@ -60,6 +69,39 @@ const MAX_CANDIDATE_SPANS = 8; /** `name.ext` shape with a plausible source extension (no slash required). */ const DOTTED_BASENAME = /^[^\s/\\]+\.[A-Za-z][A-Za-z0-9]{0,7}$/; +/** + * Extension-less kebab basename (`background-image-table`). Hyphens are + * illegal in identifiers, so consuming these tokens can never steal one from + * the named-symbol seeder; ≥2 segments keeps single words out. + */ +const KEBAB_BASENAME = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+$/; + +/** A basename's last dot-extension, same shape DOTTED_BASENAME accepts. */ +const LAST_EXTENSION = /\.[A-Za-z][A-Za-z0-9]{0,7}$/; + +/** + * Lowercased basename stems of the hyphen-named indexed files, stem → paths. + * A stem drops only the LAST extension (`a-b.module.scss` → `a-b.module`), so + * a bare kebab token can't accidentally pin a same-named stylesheet or + * `.d.ts` sibling of the source file it names; an extension-less basename + * (`pre-commit`) is its own stem. Hyphen-free basenames are skipped — a + * KEBAB_BASENAME token can never equal one, and the filter keeps the map + * near-empty in repos that don't name files this way. + */ +function buildBasenameStems(indexedPaths: readonly string[]): Map { + const stems = new Map(); + for (const p of indexedPaths) { + const basename = p.slice(Math.max(p.lastIndexOf('/'), p.lastIndexOf('\\')) + 1); + if (!basename.includes('-')) continue; + const stem = basename.replace(LAST_EXTENSION, '').toLowerCase(); + if (!stem) continue; + const existing = stems.get(stem); + if (existing) existing.push(p); + else stems.set(stem, [p]); + } + return stems; +} + /** * Strip prose punctuation wrapped around a token without eating punctuation * that is PART of the path: quotes/backticks always strip; a trailing `)`/`]` @@ -204,6 +246,38 @@ export function extractQueryPaths( // leave the token for the normal matching pipeline. } + // Second pass — extension-less kebab basenames. `background-image-table` + // opens no door above (no slash, no dotted tail), the hyphens disqualify it + // from the named-symbol seeder downstream, and FTS shreds it into the most + // common words in a kebab-cased repo (`background`, `image`, `table`) — + // which admit look-alike SIBLINGS that crowd out the named file. Resolution + // stays the detector: a token pins only when its whole lowercased form is + // the stem of an indexed basename. Two deliberate asymmetries vs the first + // pass: prose that resolves to nothing (`non-blocking`, `cross-call`) is + // LEFT IN the query — unlike a slashed span it may be legitimate wording, + // so it keeps feeding FTS and is not reported as an unresolved path — and a + // stem hotter than maxMatchesPerSpan is likewise left alone (pinning half a + // monorepo off one hot name trades precision the wrong way; a directory + // segment, which the first pass handles, disambiguates). Runs after the + // slashed/dotted pass so explicit paths win the shared maxPins budget, and + // examines every remaining token: lookups are O(1) map hits, so the + // scan-cost rationale behind MAX_CANDIDATE_SPANS doesn't apply. + let basenameStems: Map | null = null; + for (let i = 0; i < tokens.length && pinned.length < maxPins; i++) { + if (consumed.has(i)) continue; + const stripped = stripWrapping(tokens[i]!); + if (stripped.length < 4 || !KEBAB_BASENAME.test(stripped)) continue; + basenameStems ??= buildBasenameStems(indexedPaths); + const matches = basenameStems.get(stripped.toLowerCase()); + if (!matches || matches.length > maxMatchesPerSpan) continue; + consumed.add(i); + for (const m of matches) { + if (pinnedSeen.has(m) || pinned.length >= maxPins) continue; + pinnedSeen.add(m); + pinned.push(m); + } + } + if (consumed.size === 0) return passthrough; return { strippedQuery: tokens.filter((_, i) => !consumed.has(i)).join(' '), From 340d4b033ebcbf7b8010408829eff16c6c8c0a9a Mon Sep 17 00:00:00 2001 From: Max Hsu Date: Sun, 23 Aug 2026 00:53:26 +0800 Subject: [PATCH 07/27] fix(swift): remove catastrophic backtracking in Vapor route regex (#1547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arg-list group `(?:[^,()]+,\s*)*` was ambiguous: the trailing `\s*` and the next iteration's `[^,()]+` could both claim the same run of spaces, so a `.METHOD(...)` call with many comma-separated args that never reaches `use:` forced an exponential search. Measured on `app.get(arg0: value0, ...)`: 40ms at 20 args, 647ms at 24, 41.7s at 30, and no result after 120s at 60. Anchoring each repetition at a comma (`(?:[^,()]+,)*\s*`) makes the split unique — `,` is outside the char class, so there is nothing to re-partition. Same input is now 0.09ms at 1000 args. Match behaviour is unchanged: all four capture groups are identical on 18 hand-written Vapor route shapes (no args, single/multi path segments, `X.parameter`, multi-line calls, Environment.get non-matches) and on 200k fuzzed inputs. Fixes #1544 --- __tests__/frameworks.test.ts | 41 ++++++++++++++++++++++++++++++ src/resolution/frameworks/swift.ts | 12 ++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/__tests__/frameworks.test.ts b/__tests__/frameworks.test.ts index cc7e3555f..6064706d7 100644 --- a/__tests__/frameworks.test.ts +++ b/__tests__/frameworks.test.ts @@ -1472,6 +1472,47 @@ func boot(routes: RoutesBuilder) throws { const { nodes } = vaporResolver.extract!('configure.swift', src); expect(nodes).toHaveLength(0); }); + + // A `.METHOD(...)` call with many comma-separated args and no `use:` used to + // make the route regex backtrack exponentially (60 args hung for minutes). + it('does not backtrack exponentially on a long arg list without use:', () => { + const args = Array.from({ length: 60 }, (_, i) => `arg${i}: value${i}`).join(', '); + const src = `app.get(${args})\n`; + const start = performance.now(); + const { nodes } = vaporResolver.extract!('routes.swift', src); + const elapsed = performance.now() - start; + expect(nodes).toHaveLength(0); + expect(elapsed).toBeLessThan(250); + }); + + it('still parses every Vapor route shape after the arg-list rewrite', () => { + const src = ` +admin.get(use: self.list) +app.get("users", use: listUsers) +router.post("users", User.parameter, "edit", use: UserController.edit) +app.patch(":id" , "meta" , use: update) +app.get( + "multi", + "line", + use: multiLine +) +`; + const { nodes, references } = vaporResolver.extract!('routes.swift', src); + expect(nodes.map((n) => n.name)).toEqual([ + 'GET /', + 'GET /users', + 'POST /users/edit', + 'PATCH /:id/meta', + 'GET /multi/line', + ]); + expect(references.map((r) => r.referenceName)).toEqual([ + 'list', + 'listUsers', + 'edit', + 'update', + 'multiLine', + ]); + }); }); import { reactResolver } from '../src/resolution/frameworks/react'; diff --git a/src/resolution/frameworks/swift.ts b/src/resolution/frameworks/swift.ts index 0dd1513aa..5e1bf42cd 100644 --- a/src/resolution/frameworks/swift.ts +++ b/src/resolution/frameworks/swift.ts @@ -367,7 +367,17 @@ export const vaporResolver: FrameworkResolver = { // (`BlogUser.parameter`, `:id`, a path constant) so accept any comma-separated // args before `use:` — the label keeps only the string parts. `use:` // discriminates a real route from Environment.get("X")/req.parameters.get("X"). - const routeRegex = /\b(\w+)\.(get|post|put|patch|delete|head|options)\s*\(\s*((?:[^,()]+,\s*)*)use:\s*([A-Za-z_][\w.]*)/g; + // Each arg repetition must end at a comma, and `,` is outside the char class, + // so the split is unique and matching stays linear. The earlier + // `(?:[^,()]+,\s*)*` was ambiguous — the trailing `\s*` and the next + // iteration's `[^,()]+` could both claim the same spaces — which backtracked + // exponentially on a long arg list that never reaches `use:`. + // The tail is `\s*` rather than a lazy `[^,()]*?` on purpose: both are + // linear, but the lazy form drops the "`use:` is preceded by a comma" + // requirement and widens the match set — `req.get(foo.use: bar)` would then + // be indexed as a route (groups `["req","get","foo.","bar"]`) where both + // this pattern and the original match nothing. + const routeRegex = /\b(\w+)\.(get|post|put|patch|delete|head|options)\s*\(\s*((?:[^,()]+,)*\s*)use:\s*([A-Za-z_][\w.]*)/g; let match: RegExpExecArray | null; while ((match = routeRegex.exec(safe)) !== null) { const [, receiver, method, segsStr, handlerExpr] = match; From cc9ce09256a8e824f93e00a2af302bd33ae3360d Mon Sep 17 00:00:00 2001 From: Max Hsu Date: Sun, 23 Aug 2026 00:53:33 +0800 Subject: [PATCH 08/27] fix(extraction): detect untracked files inside untracked directories (#1213) (#1215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git status --porcelain collapses an entirely-untracked directory into a single '?? dir/' entry. collectGitStatus only recurses into such dirs to find embedded git repos, so source files in a plain untracked directory were never surfaced to sync — 'codegraph sync' reported 'Already up to date' and the watcher missed them too. Add -uall so git lists individual untracked files. Nested untracked git repos still collapse to '?? repo/' even with -uall (git never crosses a repo boundary), so the embedded-repo recursion is unaffected. Export getGitChangedFiles and add regression tests for both the plain untracked-directory case and the embedded-repo recursion (no -uall regression). Root-cause analysis and fix suggested by the reporter in #1213. --- __tests__/git-changed-untracked-dir.test.ts | 68 +++++++++++++++++++++ src/extraction/index.ts | 10 ++- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 __tests__/git-changed-untracked-dir.test.ts diff --git a/__tests__/git-changed-untracked-dir.test.ts b/__tests__/git-changed-untracked-dir.test.ts new file mode 100644 index 000000000..699c01ec2 --- /dev/null +++ b/__tests__/git-changed-untracked-dir.test.ts @@ -0,0 +1,68 @@ +/** + * Regression test for #1213: `codegraph sync` silently skips untracked files + * that live inside an untracked directory. + * + * `git status --porcelain` collapses an entirely-untracked directory into a + * single `?? frontend/` entry. getGitChangedFiles must still surface the source + * files inside it (via `-uall`) rather than dropping the whole directory. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { getGitChangedFiles } from '../src/extraction/index'; + +function git(cwd: string, args: string[]): void { + execFileSync('git', args, { cwd, stdio: 'pipe' }); +} + +describe('getGitChangedFiles — untracked directories (#1213)', () => { + const dirs: string[] = []; + + function makeRepo(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-1213-')); + dirs.push(dir); + git(dir, ['init']); + git(dir, ['config', 'user.email', 'test@example.com']); + git(dir, ['config', 'user.name', 'test']); + fs.writeFileSync(path.join(dir, 'root.js'), 'function foo() {}\n'); + git(dir, ['add', 'root.js']); + git(dir, ['commit', '-m', 'init']); + return dir; + } + + afterEach(() => { + while (dirs.length) { + fs.rmSync(dirs.pop()!, { recursive: true, force: true }); + } + }); + + it('detects source files inside a fully-untracked directory', () => { + const dir = makeRepo(); + fs.mkdirSync(path.join(dir, 'frontend')); + fs.writeFileSync(path.join(dir, 'frontend', 'app.js'), 'function bar() {}\n'); + + const changes = getGitChangedFiles(dir); + + expect(changes).not.toBeNull(); + expect(changes!.added).toContain('frontend/app.js'); + }); + + it('still recurses into an untracked embedded git repo (no -uall regression)', () => { + // `-uall` must not break the embedded-repo path: git collapses a nested + // repo to `?? embedded/` regardless of `-uall`, so its files are only + // reachable through collectGitStatus's recursion. + const dir = makeRepo(); + const embedded = path.join(dir, 'embedded'); + fs.mkdirSync(embedded); + git(embedded, ['init']); + fs.writeFileSync(path.join(embedded, 'inner.js'), 'function baz() {}\n'); + + const changes = getGitChangedFiles(dir); + + expect(changes).not.toBeNull(); + expect(changes!.added).toContain('embedded/inner.js'); + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 2b61636b6..45807c5c6 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -1104,7 +1104,7 @@ interface GitChanges { * case this cannot see (the child status that would report the deletions is gone * with it); a full `codegraph index` reconciles that. */ -function getGitChangedFiles(rootDir: string): GitChanges | null { +export function getGitChangedFiles(rootDir: string): GitChanges | null { try { const changes: GitChanges = { modified: [], added: [], deleted: [] }; // Custom extension → language overrides from the project's codegraph.json, @@ -1120,7 +1120,13 @@ function getGitChangedFiles(rootDir: string): GitChanges | null { function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, overrides?: Record, includeIgnored: Ignore | null = null, exclude: Ignore | null = null): void { const output = execFileSync( 'git', - ['status', '--porcelain', '--no-renames'], + // `-uall` lists individual untracked files instead of collapsing an + // entirely-untracked directory into one `?? dir/` entry, which would + // otherwise be dropped here (only embedded git repos are recursed into + // below). Nested untracked git repos still collapse to `?? repo/` even + // with `-uall` — git never crosses a repo boundary — so the recursion + // still handles them. (#1213) + ['status', '--porcelain', '--no-renames', '-uall'], { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true } ); From a74029105a8b3ff95d76f754d4d8d4097d948614 Mon Sep 17 00:00:00 2001 From: Max Hsu Date: Sun, 23 Aug 2026 00:53:36 +0800 Subject: [PATCH 09/27] fix(resolution): resolve ES imports targeting .xsjs/.xsjslib files (#556) (#594) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extraction half of #556 — indexing `.xsjs` / `.xsjslib` as JavaScript — already landed on main via #654. This PR is now scoped to the remaining resolution gap: the JS import-resolution list did not include the SAP HANA extensions, so an extensionless `import { x } from './helpers'` in a `.xsjs` file resolved to nothing and the cross-file call edge was dropped. Add `.xsjs` / `.xsjslib` to the `javascript` entry in EXTENSION_RESOLUTION so those imports resolve to their target file and `codegraph_callers` / `codegraph_impact` see the edge. One resolution test covers the .xsjs -> .xsjslib import; the now-redundant extraction/detection tests were dropped (covered by #654). --- CHANGELOG.md | 1 + __tests__/extraction.test.ts | 48 +++++++++++++++++++++++++++++++ src/resolution/import-resolver.ts | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aebf1653d..e8f98006c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -409,6 +409,7 @@ Full details in the entries below. - The `codegraph_search` tool's `kind: "type"` filter — a value its own schema advertises — silently matched nothing; it now correctly finds type aliases. The `codegraph_explore` tool's parameter guidance also no longer suggests running `codegraph_search` first, which contradicted explore's call-it-first design and cost agents an extra round-trip. - Symbols defined in Svelte and Vue `