diff --git a/src/commands/import.js b/src/commands/import.js
index c045ffd..58a32ae 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -3539,13 +3539,18 @@ async function resolveDocsBaseUrl(sourceUrl) {
}
}
+ const soft404Title = candidates.some((c) => c.kind === 'path') ? await fetchSoft404Title(sourceUrl.origin) : null
const existence = await Promise.all(
llmsHits.map(async ({ c, res }) => {
if (res.error && c.kind === 'subdomain') return { c, exists: false }
- return { c, exists: await docsRouteResolves(c) }
+ return { c, ...(await docsRouteResolves(c, soft404Title)) }
}),
)
- for (const { c, exists } of existence) {
+ for (const { c, exists, soft404 } of existence) {
+ if (soft404) {
+ styles.info(styles.dim(` ${c.url.href} → indistinguishable from a nonexistent path (SPA catch-all) — skipping`))
+ continue
+ }
if (exists) {
styles.info(styles.dim(` ${c.url.href} → resolves (no llms.txt)`))
return { url: c.url, kind: c.kind, hasLlms: false }
@@ -3570,20 +3575,54 @@ function inCandidateScope(candidate, finalUrl) {
return false
}
if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname
- return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase())
+ const strip = (p) => p.toLowerCase().replace(/\/+$/, '')
+ const finalPath = strip(final.pathname)
+ const candidatePath = strip(candidate.url.pathname)
+ return finalPath === candidatePath || finalPath.startsWith(`${candidatePath}/`)
+}
+
+function extractHtmlTitle(html) {
+ const match = /
]*>([^<]*)<\/title>/i.exec(html)
+ if (!match) return null
+ const title = match[1].replace(/\s+/g, ' ').trim().toLowerCase()
+ return title || null
+}
+
+/**
+ * Fetch a guaranteed-nonexistent path and capture its page title. On SPA
+ * catch-all sites every unknown path returns 200 with the same shell page;
+ * that title becomes the soft-404 baseline. Returns null when the site 404s
+ * properly (no baseline needed) or on any fetch failure.
+ */
+async function fetchSoft404Title(origin) {
+ const probeUrl = `${origin}/readme-cli-nonexistent-${Math.random().toString(36).slice(2)}`
+ try {
+ const res = await fetch(probeUrl, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+ if (!res.ok) return null
+ return extractHtmlTitle(await res.text())
+ } catch {
+ return null
+ }
}
/**
* Does a well-known docs route exist and stay in scope? Follows redirects and
* rejects anything that lands outside the candidate (a `docs.` subdomain that
- * bounces to marketing, a `/docs/` that 302s home). DNS failures → false.
+ * bounces to marketing, a `/docs/` that 302s home). When a soft-404 baseline
+ * title is known, a path candidate serving that same title is a catch-all
+ * response, not real docs. DNS failures → `{ exists: false }`.
*/
-async function docsRouteResolves(candidate) {
+async function docsRouteResolves(candidate, soft404Title) {
try {
const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
- return res.ok && inCandidateScope(candidate, res.url)
+ if (!res.ok || !inCandidateScope(candidate, res.url)) return { exists: false }
+ if (candidate.kind === 'path' && soft404Title) {
+ const title = extractHtmlTitle(await res.text())
+ if (title === soft404Title) return { exists: false, soft404: true }
+ }
+ return { exists: true }
} catch {
- return false
+ return { exists: false }
}
}
@@ -4698,7 +4737,7 @@ function makeIconPicker() {
}
}
-export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl }
+export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl, resolveDocsBaseUrl, inCandidateScope }
function formatDuration(ms) {
const safe = Math.max(0, ms)
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index bf031b4..0824f78 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -134,3 +134,59 @@ test('resolveRedirectedSourceUrl returns null when fetch throws', async () => {
}
assert.equal(await __test__.resolveRedirectedSourceUrl(new URL('https://example.com/')), null)
})
+
+function mockDocsProbeFetch(files, { catchAllBody } = {}) {
+ globalThis.fetch = async (url) => {
+ const href = String(url)
+ if (new URL(href).hostname !== 'example.com') throw new Error('getaddrinfo ENOTFOUND')
+ const entry = files[href]
+ if (entry) return { ok: true, status: 200, url: entry.finalUrl || href, text: async () => entry.body || '' }
+ if (catchAllBody && !href.endsWith('/llms.txt')) return { ok: true, status: 200, url: href, text: async () => catchAllBody }
+ return { ok: false, status: 404, url: href, text: async () => '' }
+ }
+}
+
+test('resolveDocsBaseUrl adopts a path route whose trailing slash gets stripped by a redirect', async () => {
+ mockDocsProbeFetch({
+ 'https://example.com/docs/': { finalUrl: 'https://example.com/docs', body: 'API Docs' },
+ })
+ const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com'))
+ assert.equal(result?.url.href, 'https://example.com/docs/')
+ assert.equal(result?.kind, 'path')
+ assert.equal(result?.hasLlms, false)
+})
+
+test('resolveDocsBaseUrl rejects a path route that redirects to the homepage', async () => {
+ mockDocsProbeFetch({
+ 'https://example.com/docs/': { finalUrl: 'https://example.com/' },
+ })
+ assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null)
+})
+
+test('resolveDocsBaseUrl skips SPA catch-all routes and adopts the route with distinct content', async () => {
+ mockDocsProbeFetch(
+ {
+ 'https://example.com/developers/': { finalUrl: 'https://example.com/developers', body: 'API Documentation' },
+ },
+ { catchAllBody: 'Example - Empowering' },
+ )
+ const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com'))
+ assert.equal(result?.url.href, 'https://example.com/developers/')
+ assert.equal(result?.kind, 'path')
+ assert.equal(result?.hasLlms, false)
+})
+
+test('resolveDocsBaseUrl returns null when every candidate fails', async () => {
+ mockDocsProbeFetch({})
+ assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null)
+})
+
+test('inCandidateScope normalizes trailing slashes and enforces segment boundaries', () => {
+ const candidate = { kind: 'path', url: new URL('https://example.com/docs/') }
+ assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs'), true)
+ assert.equal(__test__.inCandidateScope(candidate, 'https://www.example.com/docs'), true)
+ assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs/llms.txt'), true)
+ assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/DOCS//'), true)
+ assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docsomething'), false)
+ assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/'), false)
+})