fix(import): homepage docs-route probe trailing slash + SPA catch-all

src/commands/import.js

@@ -3539,13 +3539,18 @@ async function resolveDocsBaseUrl(sourceUrl) {
     }
   }
 
+  const soft404Title = candidates.some((c) => c.kind === 'path') ? await fetchSoft404Title(sourceUrl.origin) : null
   const existence = await Promise.all(
     llmsHits.map(async ({ c, res }) => {
       if (res.error && c.kind === 'subdomain') return { c, exists: false }
-      return { c, exists: await docsRouteResolves(c) }
+      return { c, ...(await docsRouteResolves(c, soft404Title)) }
     }),
   )
-  for (const { c, exists } of existence) {
+  for (const { c, exists, soft404 } of existence) {
+    if (soft404) {
+      styles.info(styles.dim(`  ${c.url.href} → indistinguishable from a nonexistent path (SPA catch-all) — skipping`))
+      continue
+    }
     if (exists) {
       styles.info(styles.dim(`  ${c.url.href} → resolves (no llms.txt)`))
       return { url: c.url, kind: c.kind, hasLlms: false }
@@ -3570,20 +3575,54 @@ function inCandidateScope(candidate, finalUrl) {
     return false
   }
   if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname
-  return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase())
+  const strip = (p) => p.toLowerCase().replace(/\/+$/, '')
+  const finalPath = strip(final.pathname)
+  const candidatePath = strip(candidate.url.pathname)
+  return finalPath === candidatePath || finalPath.startsWith(`${candidatePath}/`)
+}
+
+function extractHtmlTitle(html) {
+  const match = /<title[^>]*>([^<]*)<\/title>/i.exec(html)
+  if (!match) return null
+  const title = match[1].replace(/\s+/g, ' ').trim().toLowerCase()
+  return title || null
+}
+
+/**
+ * Fetch a guaranteed-nonexistent path and capture its page title. On SPA
+ * catch-all sites every unknown path returns 200 with the same shell page;
+ * that title becomes the soft-404 baseline. Returns null when the site 404s
+ * properly (no baseline needed) or on any fetch failure.
+ */
+async function fetchSoft404Title(origin) {
+  const probeUrl = `${origin}/readme-cli-nonexistent-${Math.random().toString(36).slice(2)}`
+  try {
+    const res = await fetch(probeUrl, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok) return null
+    return extractHtmlTitle(await res.text())
+  } catch {
+    return null
+  }
 }
 
 /**
  * Does a well-known docs route exist and stay in scope? Follows redirects and
  * rejects anything that lands outside the candidate (a `docs.` subdomain that
- * bounces to marketing, a `/docs/` that 302s home). DNS failures → false.
+ * bounces to marketing, a `/docs/` that 302s home). When a soft-404 baseline
+ * title is known, a path candidate serving that same title is a catch-all
+ * response, not real docs. DNS failures → `{ exists: false }`.
  */
-async function docsRouteResolves(candidate) {
+async function docsRouteResolves(candidate, soft404Title) {
   try {
     const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
-    return res.ok && inCandidateScope(candidate, res.url)
+    if (!res.ok || !inCandidateScope(candidate, res.url)) return { exists: false }
+    if (candidate.kind === 'path' && soft404Title) {
+      const title = extractHtmlTitle(await res.text())
+      if (title === soft404Title) return { exists: false, soft404: true }
+    }
+    return { exists: true }
   } catch {
-    return false
+    return { exists: false }
   }
 }
 
@@ -4698,7 +4737,7 @@ function makeIconPicker() {
   }
 }
 
-export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl }
+export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl, resolveDocsBaseUrl, inCandidateScope }
 
 function formatDuration(ms) {
   const safe = Math.max(0, ms)

src/commands/import.test.js

@@ -134,3 +134,59 @@ test('resolveRedirectedSourceUrl returns null when fetch throws', async () => {
   }
   assert.equal(await __test__.resolveRedirectedSourceUrl(new URL('https://example.com/')), null)
 })
+
+function mockDocsProbeFetch(files, { catchAllBody } = {}) {
+  globalThis.fetch = async (url) => {
+    const href = String(url)
+    if (new URL(href).hostname !== 'example.com') throw new Error('getaddrinfo ENOTFOUND')
+    const entry = files[href]
+    if (entry) return { ok: true, status: 200, url: entry.finalUrl || href, text: async () => entry.body || '' }
+    if (catchAllBody && !href.endsWith('/llms.txt')) return { ok: true, status: 200, url: href, text: async () => catchAllBody }
+    return { ok: false, status: 404, url: href, text: async () => '' }
+  }
+}
+
+test('resolveDocsBaseUrl adopts a path route whose trailing slash gets stripped by a redirect', async () => {
+  mockDocsProbeFetch({
+    'https://example.com/docs/': { finalUrl: 'https://example.com/docs', body: '<title>API Docs</title>' },
+  })
+  const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com'))
+  assert.equal(result?.url.href, 'https://example.com/docs/')
+  assert.equal(result?.kind, 'path')
+  assert.equal(result?.hasLlms, false)
+})
+
+test('resolveDocsBaseUrl rejects a path route that redirects to the homepage', async () => {
+  mockDocsProbeFetch({
+    'https://example.com/docs/': { finalUrl: 'https://example.com/' },
+  })
+  assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null)
+})
+
+test('resolveDocsBaseUrl skips SPA catch-all routes and adopts the route with distinct content', async () => {
+  mockDocsProbeFetch(
+    {
+      'https://example.com/developers/': { finalUrl: 'https://example.com/developers', body: '<title>API Documentation</title>' },
+    },
+    { catchAllBody: '<title>Example - Empowering</title>' },
+  )
+  const result = await __test__.resolveDocsBaseUrl(new URL('https://example.com'))
+  assert.equal(result?.url.href, 'https://example.com/developers/')
+  assert.equal(result?.kind, 'path')
+  assert.equal(result?.hasLlms, false)
+})
+
+test('resolveDocsBaseUrl returns null when every candidate fails', async () => {
+  mockDocsProbeFetch({})
+  assert.equal(await __test__.resolveDocsBaseUrl(new URL('https://example.com')), null)
+})
+
+test('inCandidateScope normalizes trailing slashes and enforces segment boundaries', () => {
+  const candidate = { kind: 'path', url: new URL('https://example.com/docs/') }
+  assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs'), true)
+  assert.equal(__test__.inCandidateScope(candidate, 'https://www.example.com/docs'), true)
+  assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docs/llms.txt'), true)
+  assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/DOCS//'), true)
+  assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docsomething'), false)
+  assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/'), false)
+})