diff --git a/src/commands/import.js b/src/commands/import.js index c045ffd..58a32ae 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3539,13 +3539,18 @@ async function resolveDocsBaseUrl(sourceUrl) { } } + const soft404Title = candidates.some((c) => c.kind === 'path') ? await fetchSoft404Title(sourceUrl.origin) : null const existence = await Promise.all( llmsHits.map(async ({ c, res }) => { if (res.error && c.kind === 'subdomain') return { c, exists: false } - return { c, exists: await docsRouteResolves(c) } + return { c, ...(await docsRouteResolves(c, soft404Title)) } }), ) - for (const { c, exists } of existence) { + for (const { c, exists, soft404 } of existence) { + if (soft404) { + styles.info(styles.dim(` ${c.url.href} → indistinguishable from a nonexistent path (SPA catch-all) — skipping`)) + continue + } if (exists) { styles.info(styles.dim(` ${c.url.href} → resolves (no llms.txt)`)) return { url: c.url, kind: c.kind, hasLlms: false } @@ -3570,20 +3575,54 @@ function inCandidateScope(candidate, finalUrl) { return false } if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname - return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase()) + const strip = (p) => p.toLowerCase().replace(/\/+$/, '') + const finalPath = strip(final.pathname) + const candidatePath = strip(candidate.url.pathname) + return finalPath === candidatePath || finalPath.startsWith(`${candidatePath}/`) +} + +function extractHtmlTitle(html) { + const match = /]*>([^<]*)<\/title>/i.exec(html) + if (!match) return null + const title = match[1].replace(/\s+/g, ' ').trim().toLowerCase() + return title || null +} + +/** + * Fetch a guaranteed-nonexistent path and capture its page title. On SPA + * catch-all sites every unknown path returns 200 with the same shell page; + * that title becomes the soft-404 baseline. Returns null when the site 404s + * properly (no baseline needed) or on any fetch failure. + */ +async function fetchSoft404Title(origin) { + const probeUrl = `${origin}/readme-cli-nonexistent-${Math.random().toString(36).slice(2)}` + try { + const res = await fetch(probeUrl, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } }) + if (!res.ok) return null + return extractHtmlTitle(await res.text()) + } catch { + return null + } } /** * Does a well-known docs route exist and stay in scope? Follows redirects and * rejects anything that lands outside the candidate (a `docs.` subdomain that - * bounces to marketing, a `/docs/` that 302s home). DNS failures → false. + * bounces to marketing, a `/docs/` that 302s home). When a soft-404 baseline + * title is known, a path candidate serving that same title is a catch-all + * response, not real docs. DNS failures → `{ exists: false }`. */ -async function docsRouteResolves(candidate) { +async function docsRouteResolves(candidate, soft404Title) { try { const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } }) - return res.ok && inCandidateScope(candidate, res.url) + if (!res.ok || !inCandidateScope(candidate, res.url)) return { exists: false } + if (candidate.kind === 'path' && soft404Title) { + const title = extractHtmlTitle(await res.text()) + if (title === soft404Title) return { exists: false, soft404: true } + } + return { exists: true } } catch { - return false + return { exists: false } } } @@ -4698,7 +4737,7 @@ function makeIconPicker() { } } -export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl } +export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl, resolveDocsBaseUrl, inCandidateScope } function formatDuration(ms) { const safe = Math.max(0, ms) diff --git a/src/commands/import.test.js b/src/commands/import.test.js index bf031b4..0824f78 100644 --- a/src/commands/import.test.js +++ b/src/commands/import.test.js @@ -134,3 +134,59 @@ test('resolveRedirectedSourceUrl returns null when fetch throws', async () => { } assert.equal(await __test__.resolveRedirectedSourceUrl(new URL('https://example.com/')), null) }) + +function mockDocsProbeFetch(files, { catchAllBody } = {}) { + globalThis.fetch = async (url) => { + const href = String(url) + if (new URL(href).hostname !== 'example.com') throw new Error('getaddrinfo ENOTFOUND') + const entry = files[href] + if (entry) return { ok: true, status: 200, url: entry.finalUrl || href, text: async () => entry.body || '' } + if (catchAllBody && !href.endsWith('/llms.txt')) return { ok: true, status: 200, url: href, text: async () => catchAllBody } + return { ok: false, status: 404, url: href, text: async () => '' } + } +} + +test('resolveDocsBaseUrl adopts a path route whose trailing slash gets stripped by a redirect', async () => { + mockDocsProbeFetch({ + 'https://example.com/docs/': { finalUrl: 'https://example.com/docs', body: 'API Docs' }, + }) + const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com')) + assert.equal(result?.url.href, 'https://example.com/docs/') + assert.equal(result?.kind, 'path') + assert.equal(result?.hasLlms, false) +}) + +test('resolveDocsBaseUrl rejects a path route that redirects to the homepage', async () => { + mockDocsProbeFetch({ + 'https://example.com/docs/': { finalUrl: 'https://example.com/' }, + }) + assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null) +}) + +test('resolveDocsBaseUrl skips SPA catch-all routes and adopts the route with distinct content', async () => { + mockDocsProbeFetch( + { + 'https://example.com/developers/': { finalUrl: 'https://example.com/developers', body: 'API Documentation' }, + }, + { catchAllBody: 'Example - Empowering' }, + ) + const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com')) + assert.equal(result?.url.href, 'https://example.com/developers/') + assert.equal(result?.kind, 'path') + assert.equal(result?.hasLlms, false) +}) + +test('resolveDocsBaseUrl returns null when every candidate fails', async () => { + mockDocsProbeFetch({}) + assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null) +}) + +test('inCandidateScope normalizes trailing slashes and enforces segment boundaries', () => { + const candidate = { kind: 'path', url: new URL('https://example.com/docs/') } + assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs'), true) + assert.equal(__test__.inCandidateScope(candidate, 'https://www.example.com/docs'), true) + assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs/llms.txt'), true) + assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/DOCS//'), true) + assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docsomething'), false) + assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/'), false) +})