feat: secondary manifest repository signal (CM-1393)

backend/src/osspckgs/migrations/V1788307300__package_repo_signal.sql

@@ -0,0 +1,178 @@
+-- Secondary manifest repository signal (CM-1393).
+--
+-- Extends the confidence scoring introduced in V1788307200 with the manifest field
+-- that produced a declared link. A repo URL read from a fallback field (homepage,
+-- bug_tracker) is weaker evidence than one read from the dedicated repository field,
+-- so it lands one tier lower.
+--
+-- The column defaults to 'primary', so existing rows keep their current score until
+-- the next enrichment pass writes a real signal or a rescore sweep runs.
+
+ALTER TABLE package_repos
+    ADD COLUMN IF NOT EXISTS signal text NOT NULL DEFAULT 'primary'
+        CHECK (signal IN ('primary', 'secondary'));
+
+-- Adding a parameter changes the signature, so the V1788307200 function is dropped
+-- rather than replaced — CREATE OR REPLACE would leave both overloads callable.
+DROP FUNCTION IF EXISTS package_repo_confidence(
+    text, text, text, bool, bool, bool, text, bool, bigint
+);
+
+CREATE OR REPLACE FUNCTION package_repo_confidence(
+    p_source           text,
+    p_ecosystem        text,
+    p_signal           text,
+    p_provenance       text,
+    p_archived         bool,
+    p_is_fork          bool,
+    p_disabled         bool,
+    p_host             text,
+    p_competing_github bool,
+    p_repo_id          bigint
+)
+RETURNS numeric(12, 9)
+LANGUAGE plpgsql IMMUTABLE AS $$
+DECLARE
+    base            numeric;
+    source_priority int;
+    offset_units    bigint;
+BEGIN
+    base := CASE p_source
+        WHEN 'manual'    THEN 0.99
+        WHEN 'heuristic' THEN 0.30
+        WHEN 'deps_dev'  THEN CASE p_provenance
+            WHEN 'SLSA_ATTESTATION'             THEN 0.99
+            WHEN 'RUBYGEMS_PUBLISH_ATTESTATION' THEN 0.95
+            WHEN 'PYPI_PUBLISH_ATTESTATION'     THEN 0.95
+            WHEN 'GO_ORIGIN'                    THEN 0.90
+            ELSE 0.50
+        END
+        -- maven splits off npm/cargo/the rest: POM <scm> blocks are notoriously
+        -- stale (legacy SVN URLs, org renames, dead mirrors).
+        WHEN 'declared' THEN CASE WHEN p_ecosystem = 'maven' THEN 0.80 ELSE 0.85 END
+        ELSE 0.30
+    END;
+
+    -- Signal adjusts the declared tier only. A deps.dev publish attestation already
+    -- proves the publisher–repo relationship, and manual links are operator-pinned.
+    IF p_source = 'declared' AND p_signal = 'secondary' THEN
+        base := base - 0.10;
+    END IF;
+
+    source_priority := CASE p_source
+        WHEN 'manual'    THEN 3
+        WHEN 'deps_dev'  THEN 2
+        WHEN 'declared'  THEN 1
+        ELSE 0
+    END;
+
+    IF p_disabled IS TRUE THEN
+        -- Scale proportionally so pre-disabled claim ordering is preserved across sources.
+        -- The offset uses a tighter modulo so max contribution (3*1000+999)*1e-9 ≈ 4e-6
+        -- stays below the 0.00016 minimum scaled tier gap and cannot invert source ordering.
+        base := 0.05 + LEAST(base, 0.99) * 0.004;
+        offset_units := source_priority::bigint * 1000 + COALESCE(p_repo_id, 0) % 1000;
+    ELSE
+        IF p_archived IS TRUE THEN
+            base := base - 0.20;
+        END IF;
+
+        IF p_is_fork IS TRUE THEN
+            base := base - 0.10;
+        END IF;
+
+        IF p_competing_github IS TRUE AND p_host IS NOT NULL AND p_host <> 'github' THEN
+            base := base - 0.05;
+        END IF;
+
+        base := GREATEST(base, 0.05);
+
+        -- Tie-breaker: reduces same-source collisions to the rare case where two repo IDs for
+        -- the same package are congruent mod 1,000,000. BEST_REPO_LINK_JOIN uses a secondary
+        -- ORDER BY repo_id DESC as the canonical deterministic pick when confidence ties.
+        offset_units := source_priority::bigint * 1000000 + COALESCE(p_repo_id, 0) % 1000000;
+    END IF;
+
+    RETURN LEAST(base + offset_units * 0.000000001, 0.999999999);
+END;
+$$;
+
+-- Replaced only to pass cur.signal through to the widened scoring function; the
+-- chunking, locking and keyset paging are unchanged from V1788307200.
+CREATE OR REPLACE PROCEDURE rescore_package_repo_confidence(
+    p_repo_ids   bigint[] DEFAULT NULL,
+    chunk_size   int      DEFAULT 25000,
+    INOUT applied_rows int DEFAULT 0
+)
+LANGUAGE plpgsql AS $$
+DECLARE
+    batch_rows   int;
+    updated_rows int;
+    cursor_id    bigint := 0;
+BEGIN
+    IF chunk_size IS NULL OR chunk_size <= 0 THEN
+        RAISE EXCEPTION 'rescore_package_repo_confidence: chunk_size must be positive, got %', chunk_size;
+    END IF;
+
+    -- Session-level: survives the internal COMMITs below.
+    IF NOT pg_try_advisory_lock(hashtextextended('rescore_package_repo_confidence', 0)) THEN
+        RAISE EXCEPTION 'rescore_package_repo_confidence: another execution is already in progress';
+    END IF;
+
+    applied_rows := 0;
+
+    LOOP
+            WITH batch AS (
+                SELECT pr.id
+                  FROM package_repos pr
+                 WHERE pr.id > cursor_id
+                   AND (p_repo_ids IS NULL OR pr.repo_id = ANY(p_repo_ids))
+                   -- deps_dev rows with NULL provenance were ingested before this column existed;
+                   -- skip them so the backfill does not downgrade SLSA/attestation links to 0.50.
+                   -- They will be rescored correctly once the next ingest populates provenance.
+                   AND NOT (pr.source = 'deps_dev' AND pr.provenance IS NULL)
+                 ORDER BY pr.id
+                 LIMIT chunk_size
+                   FOR UPDATE
+            ),
+            updated AS (
+                UPDATE package_repos pr
+                   SET confidence = s.confidence, verified_at = NOW()
+                  FROM batch b
+                  JOIN package_repos cur ON cur.id = b.id
+                  JOIN packages p ON p.id = cur.package_id
+                  JOIN repos r ON r.id = cur.repo_id,
+                       LATERAL (
+                         SELECT package_repo_confidence(
+                           cur.source, p.ecosystem, cur.signal, cur.provenance,
+                           r.archived, r.is_fork, r.disabled, r.host,
+                           EXISTS (
+                             SELECT 1
+                               FROM package_repos c
+                               JOIN repos cr ON cr.id = c.repo_id
+                              WHERE c.package_id = cur.package_id
+                                AND c.repo_id <> cur.repo_id
+                                AND cr.host = 'github'
+                           ),
+                           cur.repo_id
+                         ) AS confidence
+                       ) s
+                 WHERE pr.id = b.id
+                   AND s.confidence IS DISTINCT FROM cur.confidence
+                RETURNING pr.id
+            )
+            SELECT COUNT(*), COALESCE(MAX(b.id), cursor_id),
+                   (SELECT COUNT(*) FROM updated)
+              INTO batch_rows, cursor_id, updated_rows
+              FROM batch b;
+
+            applied_rows := applied_rows + updated_rows;
+
+            COMMIT;
+
+            EXIT WHEN batch_rows < chunk_size;
+        END LOOP;
+
+    PERFORM pg_advisory_unlock(hashtextextended('rescore_package_repo_confidence', 0));
+END;
+$$;

docs/adr/0021-secondary-manifest-repository-signal.md

@@ -0,0 +1,132 @@
+# ADR-0021: Secondary manifest repository signal
+
+**Date**: 2026-09-01
+**Status**: accepted
+**Deciders**: Joana Maia
+
+## Context
+
+Every registry writer only created a `package_repos` row when the ecosystem's
+canonical repository field parsed — npm `repository`, cargo `repository`,
+rubygems `source_code_uri`, NuGet `<repository>`, POM `<scm><url>`. A large
+share of packages leave that field empty while publishing the same repo URL in
+`homepage`, `bugs.url`, `projectUrl`, `bug_tracker_uri`, or the POM `<url>`, and
+those packages ended up with no repo link at all — invisible to criticality,
+blast radius, and Insights.
+
+Simply widening each writer to accept any of those fields would trade
+under-coverage for wrong links: fallback fields are free-form, so
+`https://example.com/docs/getting-started` canonicalizes into a plausible
+`owner/repo` shape without being a repository. Fallback links also should not
+rank equally with a declared one.
+[ADR-0020](./0020-package-repo-confidence-scoring.md) already reserved the
+`signal` column and its −0.10 penalty for exactly this.
+
+## Decision
+
+One shared helper, `resolveManifestRepo(candidates)`
+(`packages_worker/src/utils/resolveManifestRepo.ts`), resolves a package's repo
+from an ordered candidate list. The first candidate is the ecosystem's canonical
+field and resolves as `primary`; every later candidate resolves as `secondary`.
+The result carries `{ repo, signal }`, and each writer persists the
+returned `signal` on the link. No writer computes a confidence value.
+
+### Chains
+
+| Ecosystem | Chain |
+| --- | --- |
+| npm | `repository` → `homepage` → `bugs.url` |
+| pypi | Source/Code project URL → `Homepage` → bug tracker URL |
+| cargo | `repository` → `homepage` |
+| rubygems | `source_code_uri` → `homepage_uri` → `bug_tracker_uri` |
+| packagist | `support.source` → `homepage` |
+| nuget | `<repository>` → `projectUrl` |
+| maven | POM `<scm><url>` → POM `<url>` |
+
+### Host gate
+
+Candidates go through the shared `canonicalizeRepoUrl`. A `secondary` candidate
+is rejected when canonicalization yields `host === 'other'` — recognized VCS
+hosts only. The `primary` candidate keeps its historical behaviour and still
+accepts `other`, so existing links to self-hosted Gitea, cgit, and SVN are
+unaffected. Packagist already applied this gate locally; it is now the shared
+rule.
+
+Cargo is the exception on mechanics, not on policy: its pipeline is set-based
+SQL over a dump, so `normalizeRepos` stages both `declared_repository_url` and
+`homepage` into `repo_norm`, and a new `repo_choice` table applies the same
+first-wins-with-host-gate rule in SQL. `documentation` is not staged — it is
+almost always docs.rs, which the host gate rejects anyway.
+
+## Alternatives Considered
+
+### Alternative 1: Widen each writer's existing extractor in place
+
+- **Pros**: no new module; smallest diff per ecosystem.
+- **Cons**: seven copies of the fallback order and the host gate, which is how
+  the current per-ecosystem divergence arose in the first place; the `signal`
+  value would be derived independently in each writer.
+- **Why not**: the whole point is one rule; nine implementations of "which
+  field won" is the defect, not the fix.
+
+### Alternative 2: Accept fallback URLs on any host, like the primary field does
+
+- **Pros**: maximum coverage; no URL is discarded.
+- **Cons**: a documentation site or a marketing page with two path segments
+  becomes a repo link, creating a `repos` row and an incorrect
+  `packages_published` attribution — the exact failure this epic exists to fix.
+- **Why not**: coverage gained by inventing repos is negative value; the
+  primary field at least carries the publisher's explicit claim.
+
+### Alternative 3: Score fallback links lower directly in the writers instead of adding `signal`
+
+- **Pros**: no schema column; visible in one place.
+- **Cons**: reintroduces per-writer confidence literals, and the penalty could
+  not be retuned or audited afterwards — nothing records *why* a row scored
+  lower.
+- **Why not**: ADR-0020 makes the stored score derivable from stored evidence;
+  `signal` is that evidence.
+
+### Alternative 4: Backfill a separate pass that mines fallback fields for packages with no link
+
+- **Pros**: zero risk to the existing write paths; can be re-run at will.
+- **Cons**: a second code path that has to re-fetch or re-read every manifest,
+  and it goes stale the moment a package is re-ingested.
+- **Why not**: the data is already in hand at write time; the write path is
+  the cheapest place to fix coverage.
+
+## Consequences
+
+### Positive
+
+- Packages that only publish their repo in a secondary field now get a link,
+  and the link is honestly labelled as weaker.
+- The fallback order and the host gate exist once, so adding an ecosystem means
+  declaring a candidate list.
+- Per-run counters (`primary_field_hit`, `fallback_hit_by_field`, `no_signal`)
+  make the coverage uplift measurable against the pre-merge baseline.
+
+### Negative
+
+- Secondary links are, by construction, less certain than declared ones; some
+  will be wrong even with the host gate.
+- Maven and cargo needed local restructuring (maven's POM-specific
+  `normalizeScmUrl` feeds the same `package_repos` write path; cargo's
+  `repo_choice` applies the host-gate rule in SQL) to reach the same behaviour.
+- Row counts in `package_repos` grow, and the dedup/keep-highest path now sees
+  more competing links per package.
+
+### Risks
+
+- **A secondary link can outrank a genuine one when the declared field is
+  missing on the true repo but present on a fork.** Mitigation: ADR-0020's
+  fork and archived penalties, plus ADR-0022's ownership evidence, which
+  penalises the fork's owner mismatch far more heavily than the secondary
+  penalty.
+- **Recognized-host gating rejects legitimate self-hosted repos found in a
+  fallback field.** Accepted deliberately: an unrecognized host in a free-form
+  field carries no signal that it is a repository at all. Revisit if the
+  `no_signal` counters show a material self-hosted tail.
+- **Coverage growth is hard to attribute after the fact.** Mitigation: record
+  per-ecosystem `package_repos` row counts before merge, and compare against
+  the `fallback_hit_by_field` counters afterwards.

docs/adr/README.md

@@ -27,6 +27,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions.
 | [ADR-0018](./0018-per-client-rate-limiting-members-resolve.md) | Per-client rate limiting for `POST /members/resolve` using in-memory store                           | accepted | 2026-08-12 |
 | [ADR-0019](./0019-docker-builder-runner-libc.md)               | Same libc in Docker builder and runner                                                               | accepted | 2026-08-27 |
 | [ADR-0020](./0020-package-repo-confidence-scoring.md)          | Deterministic package→repo confidence scoring                                                        | accepted | 2026-09-01 |
+| [ADR-0021](./0021-secondary-manifest-repository-signal.md)     | Secondary manifest repository signal                                                                 | accepted | 2026-09-01 |
 
 ## Why ADRs?