From 9547a2d8473b0d146c2f2ee9c23443a8a9a6263c Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 15:17:46 +0100 Subject: [PATCH 1/9] feat: add ownership-match penalty to package_repo_confidence (CM-1394) Introduces the ownership_match column on package_repos and folds it into package_repo_confidence scoring: unmatched maintainer/namespace evidence on a declared link drops confidence by 0.25, missing evidence by 0.10, on top of the existing secondary-signal penalty. Signed-off-by: Joana Maia --- .../V1788393600__package_repo_owner_match.sql | 73 +++++++ ...8393601__no_evidence_ownership_penalty.sql | 181 ++++++++++++++++++ .../src/osspckgs/sqlFragments.ts | 2 +- .../data-access-layer/src/osspckgs/types.ts | 3 + .../src/packages/repoConfidence.test.ts | 8 +- .../src/packages/repoConfidence.ts | 14 +- .../repoConfidenceScoring.integration.test.ts | 23 ++- .../data-access-layer/src/packages/repos.ts | 14 +- 8 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 backend/src/osspckgs/migrations/V1788393600__package_repo_owner_match.sql create mode 100644 backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql diff --git a/backend/src/osspckgs/migrations/V1788393600__package_repo_owner_match.sql b/backend/src/osspckgs/migrations/V1788393600__package_repo_owner_match.sql new file mode 100644 index 0000000000..8fc73642ff --- /dev/null +++ b/backend/src/osspckgs/migrations/V1788393600__package_repo_owner_match.sql @@ -0,0 +1,73 @@ +-- Ownership-evidence matching for package→repo links (CM-1394). +-- +-- The per-package registry workers call matchOwnership() in TypeScript +-- (packages_worker/src/utils/ownershipMatch.ts). Cargo is the exception: its +-- pipeline is set-based over the crates.io dump and cannot call a parser per row, +-- so it needs the same normalization available in SQL. Both sides must stay in +-- step — the vanity suffix list and the prefix rule below mirror that module. + +ALTER TABLE package_repos + ADD COLUMN IF NOT EXISTS ownership_match text NOT NULL DEFAULT 'no_evidence' + CHECK (ownership_match IN ('matched', 'unmatched', 'no_evidence')); + +CREATE OR REPLACE FUNCTION package_repo_owner_key(p_identity text) +RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + WITH base AS ( + SELECT regexp_replace(lower(btrim(COALESCE(p_identity, ''))), '^@', '') AS v + ), + trimmed AS ( + SELECT v, substring(v from '-(?:ai|io|team|labs|oss|dev)$') AS suffix FROM base + ) + SELECT NULLIF( + regexp_replace( + CASE + WHEN suffix IS NOT NULL AND length(v) > length(suffix) + 1 + THEN left(v, length(v) - length(suffix)) + ELSE v + END, + '[^a-z0-9]', '', 'g' + ), + '' + ) + FROM trimmed; +$$; + +CREATE OR REPLACE FUNCTION package_repo_owner_match(p_repo_owner text, p_candidates text[]) +RETURNS text +LANGUAGE plpgsql IMMUTABLE AS $$ +DECLARE + owner_key text; + keys text[]; +BEGIN + owner_key := package_repo_owner_key(p_repo_owner); + IF owner_key IS NULL THEN + RETURN 'no_evidence'; + END IF; + + SELECT array_agg(k) INTO keys + FROM ( + SELECT package_repo_owner_key(c) AS k + FROM unnest(COALESCE(p_candidates, '{}'::text[])) AS c + ) t + WHERE k IS NOT NULL; + + IF keys IS NULL OR cardinality(keys) = 0 THEN + RETURN 'no_evidence'; + END IF; + + -- Prefix equality on at least 4 characters: `tokio` matches `tokio-rs`, while + -- short keys stay exact so `ab` cannot claim `abcdef`. + IF EXISTS ( + SELECT 1 + FROM unnest(keys) AS k + WHERE k = owner_key + OR (length(k) <= length(owner_key) AND length(k) >= 4 AND owner_key LIKE k || '%') + OR (length(owner_key) < length(k) AND length(owner_key) >= 4 AND k LIKE owner_key || '%') + ) THEN + RETURN 'matched'; + END IF; + + RETURN 'unmatched'; +END; +$$; diff --git a/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql new file mode 100644 index 0000000000..5997b5e47c --- /dev/null +++ b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql @@ -0,0 +1,181 @@ +-- Enable the no_evidence ownership penalty (CM-1394). +-- +-- V1788393600 added the ownership_match column; V1788307300 added the signal +-- parameter. This migration widens the scoring function to read ownership_match +-- too and activates the no_evidence branch — deferred until now because no writer +-- set the column until CM-1394 landed. + +-- Adding a parameter changes the signature, so the V1788307300 function is dropped +-- rather than replaced — CREATE OR REPLACE would leave both overloads callable. +DROP FUNCTION IF EXISTS package_repo_confidence( + text, text, text, text, bool, bool, bool, text, bool, bigint +); + +CREATE OR REPLACE FUNCTION package_repo_confidence( + p_source text, + p_ecosystem text, + p_signal text, + p_ownership_match text, + p_provenance text, + p_archived bool, + p_is_fork bool, + p_disabled bool, + p_host text, + p_competing_github bool, + p_repo_id bigint +) +RETURNS numeric(12, 9) +LANGUAGE plpgsql IMMUTABLE AS $$ +DECLARE + base numeric; + source_priority int; + offset_units bigint; +BEGIN + base := CASE p_source + WHEN 'manual' THEN 0.99 + WHEN 'heuristic' THEN 0.30 + WHEN 'deps_dev' THEN CASE p_provenance + WHEN 'SLSA_ATTESTATION' THEN 0.99 + WHEN 'RUBYGEMS_PUBLISH_ATTESTATION' THEN 0.95 + WHEN 'PYPI_PUBLISH_ATTESTATION' THEN 0.95 + WHEN 'GO_ORIGIN' THEN 0.90 + ELSE 0.50 + END + -- maven splits off npm/cargo/the rest: POM blocks are notoriously + -- stale (legacy SVN URLs, org renames, dead mirrors). + WHEN 'declared' THEN CASE WHEN p_ecosystem = 'maven' THEN 0.80 ELSE 0.85 END + ELSE 0.30 + END; + + -- Signal and ownership adjust the declared tier only. A deps.dev publish + -- attestation already proves the publisher–repo relationship, and manual links + -- are operator-pinned. + IF p_source = 'declared' THEN + IF p_signal = 'secondary' THEN + base := base - 0.10; + END IF; + + IF p_ownership_match = 'unmatched' THEN + base := base - 0.25; + ELSIF p_ownership_match = 'no_evidence' THEN + base := base - 0.10; + END IF; + END IF; + + source_priority := CASE p_source + WHEN 'manual' THEN 3 + WHEN 'deps_dev' THEN 2 + WHEN 'declared' THEN 1 + ELSE 0 + END; + + IF p_disabled IS TRUE THEN + -- Scale proportionally so pre-disabled claim ordering is preserved across sources. + -- The offset uses a tighter modulo so max contribution (3*1000+999)*1e-9 ≈ 4e-6 + -- stays below the 0.00016 minimum scaled tier gap and cannot invert source ordering. + base := 0.05 + LEAST(base, 0.99) * 0.004; + offset_units := source_priority::bigint * 1000 + COALESCE(p_repo_id, 0) % 1000; + ELSE + IF p_archived IS TRUE THEN + base := base - 0.20; + END IF; + + IF p_is_fork IS TRUE THEN + base := base - 0.10; + END IF; + + IF p_competing_github IS TRUE AND p_host IS NOT NULL AND p_host <> 'github' THEN + base := base - 0.05; + END IF; + + base := GREATEST(base, 0.05); + + -- Tie-breaker: reduces same-source collisions to the rare case where two repo IDs for + -- the same package are congruent mod 1,000,000. BEST_REPO_LINK_JOIN uses a secondary + -- ORDER BY repo_id DESC as the canonical deterministic pick when confidence ties. + offset_units := source_priority::bigint * 1000000 + COALESCE(p_repo_id, 0) % 1000000; + END IF; + + RETURN LEAST(base + offset_units * 0.000000001, 0.999999999); +END; +$$; + +-- Replaced only to pass cur.ownership_match through to the widened scoring function; +-- the chunking, locking and keyset paging are unchanged from V1788307300. +CREATE OR REPLACE PROCEDURE rescore_package_repo_confidence( + p_repo_ids bigint[] DEFAULT NULL, + chunk_size int DEFAULT 25000, + INOUT applied_rows int DEFAULT 0 +) +LANGUAGE plpgsql AS $$ +DECLARE + batch_rows int; + updated_rows int; + cursor_id bigint := 0; +BEGIN + IF chunk_size IS NULL OR chunk_size <= 0 THEN + RAISE EXCEPTION 'rescore_package_repo_confidence: chunk_size must be positive, got %', chunk_size; + END IF; + + -- Session-level: survives the internal COMMITs below. + IF NOT pg_try_advisory_lock(hashtextextended('rescore_package_repo_confidence', 0)) THEN + RAISE EXCEPTION 'rescore_package_repo_confidence: another execution is already in progress'; + END IF; + + applied_rows := 0; + + LOOP + WITH batch AS ( + SELECT pr.id + FROM package_repos pr + WHERE pr.id > cursor_id + AND (p_repo_ids IS NULL OR pr.repo_id = ANY(p_repo_ids)) + -- deps_dev rows with NULL provenance were ingested before this column existed; + -- skip them so the backfill does not downgrade SLSA/attestation links to 0.50. + -- They will be rescored correctly once the next ingest populates provenance. + AND NOT (pr.source = 'deps_dev' AND pr.provenance IS NULL) + ORDER BY pr.id + LIMIT chunk_size + FOR UPDATE + ), + updated AS ( + UPDATE package_repos pr + SET confidence = s.confidence, verified_at = NOW() + FROM batch b + JOIN package_repos cur ON cur.id = b.id + JOIN packages p ON p.id = cur.package_id + JOIN repos r ON r.id = cur.repo_id, + LATERAL ( + SELECT package_repo_confidence( + cur.source, p.ecosystem, cur.signal, cur.ownership_match, cur.provenance, + r.archived, r.is_fork, r.disabled, r.host, + EXISTS ( + SELECT 1 + FROM package_repos c + JOIN repos cr ON cr.id = c.repo_id + WHERE c.package_id = cur.package_id + AND c.repo_id <> cur.repo_id + AND cr.host = 'github' + ), + cur.repo_id + ) AS confidence + ) s + WHERE pr.id = b.id + AND s.confidence IS DISTINCT FROM cur.confidence + RETURNING pr.id + ) + SELECT COUNT(*), COALESCE(MAX(b.id), cursor_id), + (SELECT COUNT(*) FROM updated) + INTO batch_rows, cursor_id, updated_rows + FROM batch b; + + applied_rows := applied_rows + updated_rows; + + COMMIT; + + EXIT WHEN batch_rows < chunk_size; + END LOOP; + + PERFORM pg_advisory_unlock(hashtextextended('rescore_package_repo_confidence', 0)); +END; +$$; diff --git a/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts b/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts index 1c4b991893..f45d84ac0b 100644 --- a/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts +++ b/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts @@ -18,7 +18,7 @@ export const STEWARD_DISPLAY_NAME_METADATA = `CASE ELSE sa.metadata END` -// Ranking of a package's repo links. The uniqueness offset in confidence (V1788307200) +// Ranking of a package's repo links. The uniqueness offset in confidence (V1788393601) // is not injective, so repo_id is what makes the order total. Mirrored in ossPackages_enriched. export function bestRepoLinkOrderBy(alias: string): string { return `ORDER BY ${alias}.confidence DESC, ${alias}.repo_id DESC` diff --git a/services/libs/data-access-layer/src/osspckgs/types.ts b/services/libs/data-access-layer/src/osspckgs/types.ts index f46dec53e4..747b93292a 100644 --- a/services/libs/data-access-layer/src/osspckgs/types.ts +++ b/services/libs/data-access-layer/src/osspckgs/types.ts @@ -95,3 +95,6 @@ export type IDbRepoUpsert = { owner: string | null name: string | null } + +// package_repos claim types live in packages/repoConfidence.ts, next to the scoring +// function they feed. diff --git a/services/libs/data-access-layer/src/packages/repoConfidence.test.ts b/services/libs/data-access-layer/src/packages/repoConfidence.test.ts index 95fea9e0d7..ff32e34265 100644 --- a/services/libs/data-access-layer/src/packages/repoConfidence.test.ts +++ b/services/libs/data-access-layer/src/packages/repoConfidence.test.ts @@ -24,10 +24,11 @@ describe('packageRepoConfidenceLabel', () => { }) describe('packageRepoLinkClaimParams', () => { - it('defaults provenance and signal for claims that carry neither', () => { + it('defaults the signals CM-1393 and CM-1394 have not started writing yet', () => { expect(packageRepoLinkClaimParams({ source: 'declared' })).toEqual({ source: 'declared', signal: 'primary', + ownershipMatch: 'no_evidence', provenance: null, }) }) @@ -37,11 +38,13 @@ describe('packageRepoLinkClaimParams', () => { packageRepoLinkClaimParams({ source: 'deps_dev', signal: 'secondary', + ownershipMatch: 'matched', provenance: 'SLSA_ATTESTATION', }), ).toEqual({ source: 'deps_dev', signal: 'secondary', + ownershipMatch: 'matched', provenance: 'SLSA_ATTESTATION', }) }) @@ -53,6 +56,7 @@ describe('packageRepoConfidenceCall', () => { expect(sql).toContain('$(source)') expect(sql).toContain('$(signal)') expect(sql).toContain('$(provenance)') + expect(sql).toContain('$(ownershipMatch)') expect(sql).toContain('p.ecosystem') expect(sql).toContain('r.archived') }) @@ -61,10 +65,12 @@ describe('packageRepoConfidenceCall', () => { const sql = packageRepoConfidenceCall('p', 'r', { source: 'pr.source', signal: 'pr.signal', + ownershipMatch: 'pr.ownership_match', provenance: 'pr.provenance', }) expect(sql).not.toContain('$(') expect(sql).toContain('pr.signal') expect(sql).toContain('pr.provenance') + expect(sql).toContain('pr.ownership_match') }) }) diff --git a/services/libs/data-access-layer/src/packages/repoConfidence.ts b/services/libs/data-access-layer/src/packages/repoConfidence.ts index 4151c483a8..c2aca2902f 100644 --- a/services/libs/data-access-layer/src/packages/repoConfidence.ts +++ b/services/libs/data-access-layer/src/packages/repoConfidence.ts @@ -1,5 +1,6 @@ export type PackageRepoSource = 'declared' | 'deps_dev' | 'heuristic' | 'manual' export type PackageRepoSignal = 'primary' | 'secondary' +export type PackageRepoOwnershipMatch = 'matched' | 'unmatched' | 'no_evidence' export type PackageRepoConfidenceLabel = 'high' | 'medium' | 'low' export const CONFIDENCE_HIGH_THRESHOLD = 0.8 @@ -14,17 +15,20 @@ export function packageRepoConfidenceLabel(confidence: number): PackageRepoConfi export type PackageRepoLinkClaim = { source: PackageRepoSource signal?: PackageRepoSignal + ownershipMatch?: PackageRepoOwnershipMatch provenance?: string | null } export function packageRepoLinkClaimParams(claim: PackageRepoLinkClaim): { source: PackageRepoSource signal: PackageRepoSignal + ownershipMatch: PackageRepoOwnershipMatch provenance: string | null } { return { source: claim.source, signal: claim.signal ?? 'primary', + ownershipMatch: claim.ownershipMatch ?? 'no_evidence', provenance: claim.provenance ?? null, } } @@ -45,6 +49,7 @@ export function competingGithubRepoExpr(packageIdExpr: string, repoIdExpr: strin export type PackageRepoClaimExprs = { source: string signal: string + ownershipMatch: string provenance: string } @@ -53,6 +58,7 @@ export type PackageRepoClaimExprs = { export const CLAIM_FROM_PARAMS: PackageRepoClaimExprs = { source: '$(source)', signal: '$(signal)', + ownershipMatch: '$(ownershipMatch)', provenance: '$(provenance)', } @@ -60,6 +66,7 @@ export function claimFromRow(alias: string): PackageRepoClaimExprs { return { source: `${alias}.source`, signal: `${alias}.signal`, + ownershipMatch: `${alias}.ownership_match`, provenance: `${alias}.provenance`, } } @@ -71,6 +78,9 @@ export const KEEP_HIGHEST_CONFLICT_UPDATE = `source = CASE WHEN EXCLUD signal = CASE WHEN EXCLUDED.source = package_repos.source OR EXCLUDED.confidence > package_repos.confidence THEN EXCLUDED.signal ELSE package_repos.signal END, + ownership_match = CASE WHEN EXCLUDED.source = package_repos.source + OR EXCLUDED.confidence > package_repos.confidence + THEN EXCLUDED.ownership_match ELSE package_repos.ownership_match END, provenance = CASE WHEN EXCLUDED.source = package_repos.source OR EXCLUDED.confidence > package_repos.confidence THEN EXCLUDED.provenance ELSE package_repos.provenance END, @@ -79,7 +89,7 @@ export const KEEP_HIGHEST_CONFLICT_UPDATE = `source = CASE WHEN EXCLUD ELSE GREATEST(EXCLUDED.confidence, package_repos.confidence) END, verified_at = NOW()` -// The only path that may produce a package_repos confidence value (V1788307300). The caller +// The only path that may produce a package_repos confidence value (V1788393601). The caller // must have the package and repo rows joined — ecosystem and repo state are read off them. export function packageRepoConfidenceCall( packageAlias: string, @@ -90,7 +100,7 @@ export function packageRepoConfidenceCall( const competing = competingGithubExpr ?? competingGithubRepoExpr(`${packageAlias}.id`, `${repoAlias}.id`) return `package_repo_confidence( - ${claim.source}, ${packageAlias}.ecosystem, ${claim.signal}, ${claim.provenance}, + ${claim.source}, ${packageAlias}.ecosystem, ${claim.signal}, ${claim.ownershipMatch}, ${claim.provenance}, ${repoAlias}.archived, ${repoAlias}.is_fork, ${repoAlias}.disabled, ${repoAlias}.host, ${competing}, ${repoAlias}.id diff --git a/services/libs/data-access-layer/src/packages/repoConfidenceScoring.integration.test.ts b/services/libs/data-access-layer/src/packages/repoConfidenceScoring.integration.test.ts index ef146d7c6d..56f1e1636c 100644 --- a/services/libs/data-access-layer/src/packages/repoConfidenceScoring.integration.test.ts +++ b/services/libs/data-access-layer/src/packages/repoConfidenceScoring.integration.test.ts @@ -5,7 +5,7 @@ import { getDbConnection } from '@crowd/database' import type { QueryExecutor } from '../queryExecutor' import { pgpQx } from '../queryExecutor' -// Integration test: hits the running packages-db, where V1788307300 defines +// Integration test: hits the running packages-db, where V1788393601 defines // package_repo_confidence. Skipped when the DB env vars are missing so unit-test runs // in CI stay green. const HAVE_DB = @@ -19,6 +19,7 @@ type ScoreInput = { source: string ecosystem?: string signal?: string + ownershipMatch?: string provenance?: string | null archived?: boolean | null isFork?: boolean | null @@ -45,12 +46,13 @@ describe.skipIf(!HAVE_DB)('package_repo_confidence', () => { async function score(input: ScoreInput): Promise { const row = await qx.selectOne( `SELECT package_repo_confidence( - $(source), $(ecosystem), $(signal), $(provenance), + $(source), $(ecosystem), $(signal), $(ownershipMatch), $(provenance), $(archived), $(isFork), $(disabled), $(host), $(competingGithub), $(repoId) )::float8 AS score`, { ecosystem: 'npm', signal: 'primary', + ownershipMatch: 'matched', provenance: null, archived: null, isFork: null, @@ -88,6 +90,22 @@ describe.skipIf(!HAVE_DB)('package_repo_confidence', () => { ).toBeCloseTo(0.9, 2) }) + it('penalises unmatched or missing ownership evidence on declared links only', async () => { + expect(await score({ source: 'declared', ownershipMatch: 'no_evidence' })).toBeCloseTo(0.75, 2) + expect(await score({ source: 'declared', ownershipMatch: 'unmatched' })).toBeCloseTo(0.6, 2) + }) + + it('leaves attested and manual links untouched by the declared-only adjustments', async () => { + expect( + await score({ + source: 'deps_dev', + provenance: 'SLSA_ATTESTATION', + ownershipMatch: 'unmatched', + }), + ).toBeCloseTo(0.99, 2) + expect(await score({ source: 'manual', ownershipMatch: 'unmatched' })).toBeCloseTo(0.99, 2) + }) + it('stacks repo-state penalties and floors at 0.05', async () => { expect(await score({ source: 'declared', archived: true })).toBeCloseTo(0.65, 2) expect(await score({ source: 'declared', isFork: true })).toBeCloseTo(0.75, 2) @@ -98,6 +116,7 @@ describe.skipIf(!HAVE_DB)('package_repo_confidence', () => { source: 'heuristic', archived: true, isFork: true, + ownershipMatch: 'unmatched', }), ).toBeCloseTo(0.05, 2) }) diff --git a/services/libs/data-access-layer/src/packages/repos.ts b/services/libs/data-access-layer/src/packages/repos.ts index f2383164e2..15b9d72b4c 100644 --- a/services/libs/data-access-layer/src/packages/repos.ts +++ b/services/libs/data-access-layer/src/packages/repos.ts @@ -78,8 +78,14 @@ export async function removeDeclaredPackageRepo( return ['package_repos.repo_id'] } +// Confidence is never passed in — package_repo_confidence() (V1788393601) is the only +// path that produces one. Callers describe the claim (source, which manifest field it +// came from, what ownership evidence backs it) and the function scores it against the +// package's ecosystem and the repo's current state. +// // Conflict policy lives in KEEP_HIGHEST_CONFLICT_UPDATE: keep-highest across sources, -// replace on a same-source refresh. +// replace on a same-source refresh (so updated ownership evidence, e.g. +// `no_evidence` → `unmatched`, is persisted). export async function upsertPackageRepo( qx: QueryExecutor, packageId: string, @@ -100,9 +106,11 @@ export async function upsertPackageRepo( ), ins AS ( INSERT INTO package_repos ( - package_id, repo_id, source, signal, provenance, confidence, created_at + package_id, repo_id, source, signal, ownership_match, provenance, + confidence, created_at ) - SELECT $(packageId)::bigint, $(repoId)::bigint, $(source), $(signal), $(provenance), + SELECT $(packageId)::bigint, $(repoId)::bigint, $(source), $(signal), + $(ownershipMatch), $(provenance), scored.confidence, NOW() FROM scored ON CONFLICT (package_id, repo_id) DO UPDATE SET From 4be467d096bd3c3a8e5a261d2800d80241e9764e Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 15:18:00 +0100 Subject: [PATCH 2/9] feat: add ownership-evidence matching util for package repos (CM-1394) Shared matchOwnership()/repoOwnerFromCanonical() helpers used by every ecosystem writer to compare a package's namespace/maintainers against the linked repo's owner and classify the result as matched, unmatched, or no_evidence. Signed-off-by: Joana Maia --- .../utils/__tests__/ownershipMatch.test.ts | 109 ++++++++++++++++ .../src/utils/ownershipMatch.ts | 116 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 services/apps/packages_worker/src/utils/__tests__/ownershipMatch.test.ts create mode 100644 services/apps/packages_worker/src/utils/ownershipMatch.ts diff --git a/services/apps/packages_worker/src/utils/__tests__/ownershipMatch.test.ts b/services/apps/packages_worker/src/utils/__tests__/ownershipMatch.test.ts new file mode 100644 index 0000000000..44327a4b36 --- /dev/null +++ b/services/apps/packages_worker/src/utils/__tests__/ownershipMatch.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest' + +import { canonicalizeRepoUrl } from '../canonicalizeRepoUrl' +import { matchOwnership, repoOwnerFromCanonical } from '../ownershipMatch' + +describe('matchOwnership', () => { + it('matches a namespace equal to the repo owner', () => { + expect(matchOwnership({ namespace: '@vercel', repoOwner: 'vercel' })).toBe('matched') + }) + + it('matches a maintainer username when the namespace does not', () => { + expect( + matchOwnership({ namespace: 'acme', maintainers: ['seldaek'], repoOwner: 'Seldaek' }), + ).toBe('matched') + }) + + it('matches on a vanity suffix and on prefix equality', () => { + expect(matchOwnership({ namespace: 'tokio-rs', repoOwner: 'tokio' })).toBe('matched') + expect(matchOwnership({ namespace: 'langchain-ai', repoOwner: 'langchain' })).toBe('matched') + }) + + it('does not let a short key claim a longer owner by prefix', () => { + expect(matchOwnership({ namespace: 'ab', repoOwner: 'abcdef' })).toBe('unmatched') + }) + + it('matches a reverse-DNS namespace segment', () => { + expect(matchOwnership({ namespace: 'org.projectlombok', repoOwner: 'projectlombok' })).toBe( + 'matched', + ) + expect(matchOwnership({ namespace: 'io.github.resilience4j', repoOwner: 'resilience4j' })).toBe( + 'matched', + ) + }) + + it('does not match structural DNS/VCS segments against a repo owner', () => { + // `io.github.attacker` must not claim the `github` org + expect(matchOwnership({ namespace: 'io.github.attacker', repoOwner: 'github' })).toBe( + 'unmatched', + ) + // `io.github.attacker` must not prefix-match `io-github` via the joined form `iogithubattacker` + expect(matchOwnership({ namespace: 'io.github.attacker', repoOwner: 'io-github' })).toBe( + 'unmatched', + ) + // `org.foo` must not match an owner literally named `org` + expect(matchOwnership({ namespace: 'org.foo', repoOwner: 'org' })).toBe('unmatched') + // `de.github-ai.pkg` — `github-ai` normalises to `github` via vanity-suffix stripping, filtered by set membership + expect(matchOwnership({ namespace: 'de.github-ai.pkg', repoOwner: 'github' })).toBe('unmatched') + }) + + it('does not discard legitimate identity segments that share a prefix with a structural label', () => { + // `io.github.github-tools` — `github-tools` normalises to `githubtools`, which is NOT structural + // and must not be filtered even though it starts with `github` + expect(matchOwnership({ namespace: 'io.github.github-tools', repoOwner: 'github-tools' })).toBe( + 'matched', + ) + }) + + it('reports unmatched when evidence exists but nothing lines up', () => { + expect( + matchOwnership({ namespace: 'squatter', maintainers: ['nobody'], repoOwner: 'torvalds' }), + ).toBe('unmatched') + }) + + it('reports no_evidence without a repo owner', () => { + expect(matchOwnership({ namespace: 'vercel', repoOwner: null })).toBe('no_evidence') + }) + + it('reports no_evidence when the ecosystem exposes no namespace or maintainers', () => { + expect(matchOwnership({ maintainers: [null, undefined, ''], repoOwner: 'vercel' })).toBe( + 'no_evidence', + ) + }) + + it('excludes email-format maintainer strings to prevent domain-prefix false matches', () => { + // acme@example.com normalises to acmeexamplecom, which would prefix-match owner acme + expect(matchOwnership({ maintainers: ['acme@example.com'], repoOwner: 'acme' })).toBe( + 'no_evidence', + ) + // @vercel is a handle, not an email — the leading @ is stripped by normalizeIdentity + expect(matchOwnership({ maintainers: ['@vercel'], repoOwner: 'vercel' })).toBe('matched') + }) + + it('matches flat dotted namespaces like NuGet or Packagist vendors by all segments', () => { + // Microsoft.Extensions — first segment is identity-bearing, not a TLD + expect(matchOwnership({ namespace: 'Microsoft.Extensions', repoOwner: 'microsoft' })).toBe( + 'matched', + ) + // Packagist vendor with dot — foo should be a candidate + expect(matchOwnership({ namespace: 'foo.bar', repoOwner: 'foo' })).toBe('matched') + }) +}) + +describe('repoOwnerFromCanonical', () => { + it('takes the first path segment', () => { + const repo = canonicalizeRepoUrl('https://github.com/vercel/next.js') + expect(repo && repoOwnerFromCanonical(repo)).toBe('vercel') + }) + + it('takes the top-level group of a gitlab subgroup path', () => { + const repo = canonicalizeRepoUrl('https://gitlab.com/group/sub/project') + expect(repo && repoOwnerFromCanonical(repo)).toBe('group') + }) + + it('returns null for host=other where first path segment is not an owner', () => { + const repo = canonicalizeRepoUrl('https://git.sr.ht/~sircmpwn/aerc') + expect(repo?.host).toBe('other') + expect(repo && repoOwnerFromCanonical(repo)).toBeNull() + }) +}) diff --git a/services/apps/packages_worker/src/utils/ownershipMatch.ts b/services/apps/packages_worker/src/utils/ownershipMatch.ts new file mode 100644 index 0000000000..cfa769b9f2 --- /dev/null +++ b/services/apps/packages_worker/src/utils/ownershipMatch.ts @@ -0,0 +1,116 @@ +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' + +import type { CanonicalRepo } from './canonicalizeRepoUrl' + +export interface OwnershipEvidence { + // Registry namespace: npm scope, Maven groupId, packagist vendor, ... — null when the + // ecosystem has no namespace concept (cargo, rubygems, nuget). + namespace?: string | null + maintainers?: Array + repoOwner: string | null +} + +const VANITY_SUFFIXES = ['-ai', '-io', '-team', '-labs', '-oss', '-dev'] + +function normalizeIdentity(raw: string): string { + let s = raw.trim().toLowerCase() + if (!s) return '' + s = s.replace(/^@/, '') + for (const suffix of VANITY_SUFFIXES) { + if (s.endsWith(suffix) && s.length > suffix.length + 1) { + s = s.slice(0, -suffix.length) + break + } + } + return s.replace(/[^a-z0-9]/g, '') +} + +// Structural labels in reverse-DNS namespaces (TLDs, VCS hostnames) are not owner identities. +// `io.github.attacker` must not produce `github` as a candidate — only `attacker` is identity-bearing. +const STRUCTURAL_SEGMENTS = new Set([ + 'com', + 'org', + 'net', + 'io', + 'dev', + 'app', + 'co', + // common country-code TLDs that appear in Maven group IDs (e.g. uk.co.foo, au.com.bar) + 'uk', + 'au', + 'us', + 'de', + 'fr', + 'in', + 'jp', + 'cn', + 'br', + 'eu', + 'ru', + 'nl', + 'it', + 'es', + 'pl', + 'se', + 'nz', + 'za', + 'mx', + 'ar', + 'github', + 'gitlab', + 'bitbucket', + 'sourceforge', + 'codeberg', +]) + +// Reverse-DNS namespaces (Maven `org.apache.commons`) carry the owner in one of their segments. +// For multi-segment namespaces, if the first segment is a structural label (TLD like `org`, +// `com`, `io`) skip it — otherwise keep it (e.g., Packagist vendors like `foo.bar` or NuGet +// namespaces like `Microsoft.Extensions` where the first segment is identity-bearing). Flat +// scopes (`@vercel`, `tokio-rs`) are single-segment — normalise the whole string. +// normalizeIdentity already strips vanity suffixes, so `github-ai` → `github` hits the +// set-membership check directly; isSameIdentity is not needed here and would over-eagerly +// discard legitimate identities like `github-tools` (normalises to `githubtools`). +function namespaceCandidates(namespace: string): string[] { + const segments = namespace.split(/[./]/).filter(Boolean) + if (segments.length === 1) { + return [normalizeIdentity(namespace)].filter(Boolean) + } + const firstNorm = normalizeIdentity(segments[0]) + const firstIsStructural = STRUCTURAL_SEGMENTS.has(firstNorm) + return segments + .slice(firstIsStructural ? 1 : 0) + .map(normalizeIdentity) + .filter((normalized) => normalized && !STRUCTURAL_SEGMENTS.has(normalized)) +} + +function isSameIdentity(a: string, b: string): boolean { + if (!a || !b) return false + if (a === b) return true + const [short, long] = a.length <= b.length ? [a, b] : [b, a] + return short.length >= 4 && long.startsWith(short) +} + +export function matchOwnership(evidence: OwnershipEvidence): PackageRepoOwnershipMatch { + const repoOwner = evidence.repoOwner ? normalizeIdentity(evidence.repoOwner) : '' + if (!repoOwner) return 'no_evidence' + + const candidates = [ + ...(evidence.namespace ? namespaceCandidates(evidence.namespace) : []), + ...(evidence.maintainers ?? []) + .filter((m) => m && !/\S@\S/.test(m)) + .map((m) => normalizeIdentity(m!)), + ].filter(Boolean) + + if (candidates.length === 0) return 'no_evidence' + return candidates.some((c) => isSameIdentity(c, repoOwner)) ? 'matched' : 'unmatched' +} + +// GitLab subgroups make the owner the first path segment, not the second-to-last one. +// Returns null for host=other: those URLs preserve full path segments (e.g. sourceforge /p/foo/code) +// so path[0] is not the owner. +export function repoOwnerFromCanonical(repo: CanonicalRepo): string | null { + if (repo.host === 'other') return null + const path = repo.url.replace(/^https?:\/\/[^/]+\//, '').split('/') + return path.length >= 2 ? path[0] : null +} From adb406976e9b4bdaad1e1bf4ce04b4b857b2255e Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 15:18:11 +0100 Subject: [PATCH 3/9] feat: wire ownership-evidence matching into declared repo writers (CM-1394) Every ecosystem's declared-repo-link write (cargo, deps.dev, go, maven, npm, nuget, packagist, pypi) now computes ownershipMatch via the shared matchOwnership() util and persists it alongside the link, feeding the new package_repo_confidence penalty. Signed-off-by: Joana Maia --- .../apps/packages_worker/src/cargo/enrich.ts | 14 +++++- .../src/cargo/normalizeRepos.ts | 10 +++-- .../src/deps-dev/workflows/ingestRepos.ts | 34 +++++++++++--- .../apps/packages_worker/src/go/activities.ts | 44 ++++++++++++++++++- .../src/maven/runMavenEnrichmentLoop.ts | 14 ++++-- .../packages_worker/src/npm/upsertPackage.ts | 6 +++ .../src/nuget/runNuGetEnrichmentLoop.ts | 5 +++ .../__tests__/persistPackageInfo.test.ts | 4 ++ .../src/packagist/upsertPackageInfo.ts | 6 +++ .../packages_worker/src/pypi/upsertProject.ts | 5 +++ 10 files changed, 126 insertions(+), 16 deletions(-) diff --git a/services/apps/packages_worker/src/cargo/enrich.ts b/services/apps/packages_worker/src/cargo/enrich.ts index b9246fd4b0..e89dee7ac4 100644 --- a/services/apps/packages_worker/src/cargo/enrich.ts +++ b/services/apps/packages_worker/src/cargo/enrich.ts @@ -23,6 +23,7 @@ const REPO_LINK_SOURCE = 'declared' // same convention as npm/maven for manifest const CARGO_CONFIDENCE = packageRepoConfidenceCall('p', 'r', { source: '$(source)', signal: 'rc.signal', + ownershipMatch: 'om.match', provenance: 'NULL', }) @@ -236,13 +237,22 @@ export async function enrichRepos(qx: QueryExecutor): Promise ), ins AS ( INSERT INTO package_repos ( - package_id, repo_id, source, signal, provenance, confidence, created_at, verified_at + package_id, repo_id, source, signal, ownership_match, provenance, + confidence, created_at, verified_at ) - SELECT rc.package_id, r.id, $(source), rc.signal, NULL, + SELECT rc.package_id, r.id, $(source), rc.signal, om.match, NULL, s.confidence, NOW(), NOW() FROM ${STAGING_SCHEMA}.repo_choice rc JOIN repos r ON r.url = rc.repository_url JOIN packages p ON p.id = rc.package_id + CROSS JOIN LATERAL ( + SELECT package_repo_owner_match( + rc.owner, + ARRAY(SELECT em.github_login + FROM ${STAGING_SCHEMA}.enrich_maintainers em + WHERE em.package_id = rc.package_id) + ) AS match + ) om CROSS JOIN LATERAL (SELECT ${CARGO_CONFIDENCE} AS confidence) s ON CONFLICT (package_id, repo_id) DO UPDATE SET ${KEEP_HIGHEST_CONFLICT_UPDATE} diff --git a/services/apps/packages_worker/src/cargo/normalizeRepos.ts b/services/apps/packages_worker/src/cargo/normalizeRepos.ts index fd4e8e3f17..60aa2bfa52 100644 --- a/services/apps/packages_worker/src/cargo/normalizeRepos.ts +++ b/services/apps/packages_worker/src/cargo/normalizeRepos.ts @@ -2,6 +2,7 @@ import { QueryExecutor } from '@crowd/data-access-layer' import { getServiceChildLogger } from '@crowd/logging' import { CanonicalRepo, canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' +import { repoOwnerFromCanonical } from '../utils/ownershipMatch' import { STAGING_SCHEMA } from './loadDump' import { NormalizeReposResult } from './types' @@ -27,7 +28,8 @@ export async function normalizeRepos(qx: QueryExecutor): Promise r.declared), urls: batch.map((r) => r.canonical.url), hosts: batch.map((r) => r.canonical.host), + owners: batch.map((r) => repoOwnerFromCanonical(r.canonical)), }, ) } @@ -58,6 +61,7 @@ export async function normalizeRepos(qx: QueryExecutor): Promise 'github' + AND NOT (pr.source = 'deps_dev' AND pr.provenance IS NULL) + AND ${competingGithubRepoExpr('p.id', 'r.id')} + AND s.confidence IS DISTINCT FROM pr.confidence +` + const PKGREPOS_MERGE_SQL = ` WITH github_staged AS MATERIALIZED ( SELECT DISTINCT p2.id AS package_id @@ -111,10 +129,11 @@ WITH github_staged AS MATERIALIZED ( WHERE r2.host = 'github' ) INSERT INTO package_repos ( - package_id, repo_id, source, signal, provenance, confidence, verified_at, created_at + package_id, repo_id, source, signal, ownership_match, provenance, + confidence, verified_at, created_at ) SELECT DISTINCT ON (p.id, r.id) - p.id, r.id, 'deps_dev', 'primary', s.provenance, + p.id, r.id, 'deps_dev', 'primary', 'no_evidence', s.provenance, c.confidence, NOW(), NOW() FROM staging.osspckgs_package_repos_raw s JOIN packages p ON p.purl = REGEXP_REPLACE(s.purl, '@[^@]+$', '') @@ -126,6 +145,7 @@ CROSS JOIN LATERAL ( { source: `'deps_dev'`, signal: `'primary'`, + ownershipMatch: `'no_evidence'`, provenance: 's.provenance', }, `((r.host <> 'github' AND EXISTS (SELECT 1 FROM github_staged gs WHERE gs.package_id = p.id)) OR ${competingGithubRepoExpr('p.id', 'r.id')})`, @@ -274,8 +294,8 @@ export async function ingestRepos(opts: { const { rowsAffected, tableRowCounts } = await mergeStagingToTable({ jobId: pkgReposExport.jobId, - mergeSql: PKGREPOS_MERGE_SQL, - tableNames: 'package_repos', + mergeSql: isFinal ? [PKGREPOS_MERGE_SQL, PKGREPOS_RESCORE_SQL] : PKGREPOS_MERGE_SQL, + tableNames: isFinal ? ['package_repos', 'package_repos'] : 'package_repos', isFinal, priorRowsAffected: pkgRepoPriorRowsAffected, priorTableRowCounts: pkgRepoPriorTableRowCounts, diff --git a/services/apps/packages_worker/src/go/activities.ts b/services/apps/packages_worker/src/go/activities.ts index 7a044ae757..96a5060716 100644 --- a/services/apps/packages_worker/src/go/activities.ts +++ b/services/apps/packages_worker/src/go/activities.ts @@ -11,6 +11,7 @@ import { getServiceChildLogger } from '@crowd/logging' import { getGoConfig } from '../config' import { getPackagesDb } from '../db' import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' import { fetchStatus } from './pkgGoDevClient' import { fetchLatest } from './proxyClient' @@ -28,6 +29,41 @@ export interface GoScanCursor { type GoRow = { id: string; purl: string; name: string; declaredRepositoryUrl: string | null } +const GO_VCS_MODULE_HOSTS = new Set([ + 'github.com', + 'gitlab.com', + 'bitbucket.org', + 'codeberg.org', + 'gitea.com', + 'git.sr.ht', +]) + +// Only module paths rooted at a real VCS host carry the owner in their second segment. +// Vanity paths (`k8s.io/client-go`, `gopkg.in/yaml.v2`) name the package, not the owner, so +// deriving one there would produce a false `unmatched`. +function goModuleOwner(name: string): string | null { + const segments = name.split('/') + if (segments.length < 2 || !GO_VCS_MODULE_HOSTS.has(segments[0].toLowerCase())) return null + return segments[1] || null +} + +// canonicalizeRepoUrl classifies codeberg, gitea, and git.sr.ht as host='other' (unknown +// forges), so repoOwnerFromCanonical returns null for them. For Go VCS hosts specifically, +// the owner is always the first URL path segment — fall back to extracting it directly. +function goRepoOwner(canonical: ReturnType): string | null { + if (!canonical) return null + const owner = repoOwnerFromCanonical(canonical) + if (owner !== null) return owner + try { + const parsed = new URL(canonical.url) + if (!GO_VCS_MODULE_HOSTS.has(parsed.hostname)) return null + const parts = parsed.pathname.split('/').filter(Boolean) + return parts.length >= 2 ? parts[0] : null + } catch { + return null + } +} + // Two independent purl-keyset cursors — one for critical packages, one for everything else — // each ordered/paginated purely by purl so WHERE and ORDER BY always match (no gaps, no // duplicates). A single query sorted by is_critical DESC with one shared purl cursor was tried @@ -138,7 +174,13 @@ export async function enrichGoVersionsBatch( ) changedFields.push(...repoChanged) - const linkChanged = await upsertPackageRepo(t, row.id, repoId, { source: 'declared' }) + const linkChanged = await upsertPackageRepo(t, row.id, repoId, { + source: 'declared', + ownershipMatch: matchOwnership({ + namespace: goModuleOwner(row.name), + repoOwner: goRepoOwner(repoToLink), + }), + }) changedFields.push(...linkChanged) } diff --git a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts index 3193892bab..28caff0488 100644 --- a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts @@ -19,6 +19,7 @@ import type { PackageRepoSignal } from '@crowd/data-access-layer/src/packages/re import { getServiceChildLogger } from '@crowd/logging' import { getMavenConfig } from '../config' +import { OwnershipEvidence, matchOwnership } from '../utils/ownershipMatch' import { resolveManifestRepo } from '../utils/resolveManifestRepo' import { extractArtifact, getPomCacheStats, normalizeScmUrl } from './extract' @@ -59,7 +60,7 @@ type PackageRow = MavenPackageToSync // ─── Helpers ────────────────────────────────────────────────────────────────── // prettier-ignore -export async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed?: Set, signal: PackageRepoSignal = 'primary'): Promise { +export async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed?: Set, signal: PackageRepoSignal = 'primary', evidence?: Omit): Promise { if (!repositoryUrl) { const removedFields = await removeDeclaredPackageRepo(qx, String(packageId)) removedFields.forEach((f) => changed?.add(f)) @@ -76,7 +77,11 @@ export async function writeRepoLink(qx: QueryExecutor, packageId: number, reposi return } const repoId = await upsertRepo(qx, { url: repositoryUrl, ...parsed }) - const repoChanged = await upsertPackageRepo(qx, String(packageId), String(repoId), { source: 'declared', signal }) + const ownershipMatch = matchOwnership({ + ...evidence, + repoOwner: parsed.host === 'other' ? null : parsed.owner, + }) + const repoChanged = await upsertPackageRepo(qx, String(packageId), String(repoId), { source: 'declared', signal, ownershipMatch }) repoChanged.forEach((f) => changed?.add(f)) const removedFields = await removeDeclaredPackageRepo(qx, String(packageId), String(repoId)) removedFields.forEach((f) => changed?.add(f)) @@ -367,7 +372,10 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF ) declaredClearedFields.forEach((f) => changed.add(f)) - await writeRepoLink(t, packageId, repositoryUrl, changed, fallbackRepo ? 'secondary' : 'primary') + await writeRepoLink(t, packageId, repositoryUrl, changed, fallbackRepo ? 'secondary' : 'primary', { + namespace: groupId, + maintainers: allPeople.map((p) => p.username), + }) await logAuditFieldChange(t, 'maven', pkg.purl, Array.from(changed)) diff --git a/services/apps/packages_worker/src/npm/upsertPackage.ts b/services/apps/packages_worker/src/npm/upsertPackage.ts index fdae4158b9..13b91fdfa8 100644 --- a/services/apps/packages_worker/src/npm/upsertPackage.ts +++ b/services/apps/packages_worker/src/npm/upsertPackage.ts @@ -9,6 +9,7 @@ import { } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' import { @@ -93,6 +94,11 @@ export async function upsertPackage( const linkChanged = await upsertPackageRepo(t, pkgId, repoId, { source: 'declared', signal: resolvedRepo.signal, + ownershipMatch: matchOwnership({ + namespace, + maintainers: maintainers.filter((m) => m.role === 'maintainer').map((m) => m.username), + repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), + }), }) linkChanged.forEach((f) => changed.add(f)) diff --git a/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts b/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts index 945b05634c..40d7265f80 100644 --- a/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts @@ -19,6 +19,7 @@ import { import { getServiceChildLogger } from '@crowd/logging' import { getNuGetConfig } from '../config' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' import { fetchNuspec, fetchRegistration, fetchSearch } from './client' import { normalizeNuGetPackage } from './normalize' @@ -175,6 +176,10 @@ async function processPackage( const linkChanged = await upsertPackageRepo(t, packageDbId.toString(), repoId, { source: 'declared', signal: normalized.resolvedRepo.signal, + ownershipMatch: matchOwnership({ + maintainers: [...normalized.owners, ...normalized.authors], + repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), + }), }) linkChanged.forEach((f) => changed.add(f)) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index 6d0c8c31ec..8c556c1d66 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -87,6 +87,7 @@ describe('persistPackagistPackageInfo', () => { expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '55', { source: 'declared', signal: 'primary', + ownershipMatch: 'matched', }) // any stale 'declared' link pointing at a different repo is pruned in the same pass expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7', '55') @@ -114,6 +115,7 @@ describe('persistPackagistPackageInfo', () => { expect(mockRepoLink).toHaveBeenCalledWith(qx, '8', '55', { source: 'declared', signal: 'primary', + ownershipMatch: 'matched', }) }) @@ -127,6 +129,7 @@ describe('persistPackagistPackageInfo', () => { expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '99', { source: 'declared', signal: 'primary', + ownershipMatch: 'matched', }) // old link (some other repo_id) removed, new one (99) kept expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7', '99') @@ -172,6 +175,7 @@ describe('persistPackagistPackageInfo', () => { expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '55', { source: 'declared', signal: 'secondary', + ownershipMatch: 'matched', }) expect(mockUpdate).toHaveBeenCalledWith( qx, diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index fba0d7e77b..9b6c432a7f 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -11,6 +11,7 @@ import { import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' import { resolveManifestRepo } from '../utils/resolveManifestRepo' import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' @@ -88,6 +89,11 @@ export async function persistPackagistPackageInfo( const linkChanged = await upsertPackageRepo(t, id, repo.id, { source: 'declared', signal: resolvedRepo.signal, + ownershipMatch: matchOwnership({ + namespace: stats.name.split('/')[0] || null, + maintainers: stats.maintainers.map((m) => m.username), + repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), + }), }) const removedFields = await removeDeclaredPackageRepo(t, id, repo.id) changedFields.push(...repo.changedFields, ...linkChanged, ...removedFields) diff --git a/services/apps/packages_worker/src/pypi/upsertProject.ts b/services/apps/packages_worker/src/pypi/upsertProject.ts index a423d75808..0175be5879 100644 --- a/services/apps/packages_worker/src/pypi/upsertProject.ts +++ b/services/apps/packages_worker/src/pypi/upsertProject.ts @@ -10,6 +10,7 @@ import { import type { PackageRepoSignal } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' import { resolveManifestRepo } from '../utils/resolveManifestRepo' import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' @@ -103,6 +104,10 @@ export async function upsertProject( const linkChanged = await upsertPackageRepo(t, pkgId, repoId, { source: 'declared', signal: repoSignal, + ownershipMatch: matchOwnership({ + maintainers: maintainers.map((m) => m.username), + repoOwner: repoOwnerFromCanonical(repo), + }), }) linkChanged.forEach((f) => changed.add(f)) From 84af02f524a4d8cc73c3e471fb642ed8d5eb3917 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 15:18:17 +0100 Subject: [PATCH 4/9] docs: add ADR-0022 for package-repo ownership-evidence signal (CM-1394) Signed-off-by: Joana Maia --- .../0022-package-repo-ownership-evidence.md | 105 ++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 106 insertions(+) create mode 100644 docs/adr/0022-package-repo-ownership-evidence.md diff --git a/docs/adr/0022-package-repo-ownership-evidence.md b/docs/adr/0022-package-repo-ownership-evidence.md new file mode 100644 index 0000000000..0f4da1d3ed --- /dev/null +++ b/docs/adr/0022-package-repo-ownership-evidence.md @@ -0,0 +1,105 @@ +# ADR-0022: Ownership evidence for package→repo links + +**Date**: 2026-09-01 +**Status**: accepted +**Deciders**: Joana Maia + +## Context + +A package can name any repository it likes. Nothing in the ingestion path ever +checked whether the entity publishing the package has anything to do with the +entity owning the repo, so a package declaring `github.com/torvalds/linux` +produced a link indistinguishable from the kernel's own. That is how unrelated +packages inflate a repo's `packages_published` count and, through it, every +aggregate computed from those counts. + +The evidence needed for the check is already ingested: npm scopes, Maven +groupIds, packagist vendors, Go module paths, and maintainer/owner/author logins +for most registries. [ADR-0020](./0020-package-repo-confidence-scoring.md) +reserved the `ownership_match` column and priced it — `unmatched` −0.25, +`no_evidence` −0.10 — so all that is missing is a producer for the value. + +## Decision + +`matchOwnership({ namespace, maintainers, repoOwner })` +(`packages_worker/src/utils/ownershipMatch.ts`) returns `matched`, +`unmatched`, or `no_evidence`, and every declared writer calls it before +persisting a link. Namespace evidence is checked first, maintainer logins are +the fallback. `no_evidence` is a distinct outcome from `unmatched`: a registry +that exposes nothing to compare must not be treated as a failed comparison. + +### Normalisation + +Identities are compared after `normalizeIdentity`: trim, lowercase, strip a +leading `@`, strip one trailing vanity suffix from `-ai`, `-io`, `-team`, +`-labs`, `-oss`, `-dev` (only when a suffix strip leaves more than one +character), then drop all non-alphanumerics. Reverse-DNS namespaces (multi-segment, e.g. `org.projectlombok`) contribute +only their identity-bearing segments as candidates — structural labels (`io`, +`com`, `org`, `github`, etc.) are excluded, and no joined form is produced. +Flat single-segment scopes are normalised as a whole string. This means +`io.github.resilience4j` matches `resilience4j` via its identity-bearing +segment, while `io.github.attacker` cannot prefix-match a lookalike owner +such as `io-github`. Two identities match on +equality or on prefix when the shorter is at least 4 characters, which is what +makes `tokio-rs` → `tokio` and `langchain-ai` → `langchain` match while `ab` +against `abcdef` stays `unmatched`. + +### Per-ecosystem evidence + +| Ecosystem | Namespace evidence | Maintainer evidence | +| --- | --- | --- | +| npm | package scope | `maintainers` | +| pypi | — | maintainer/author names | +| packagist | vendor from `name` | maintainers | +| nuget | — | owners + authors | +| maven | groupId | developer/contributor usernames | +| cargo | — | maintainer GitHub logins | +| go | module path owner, VCS hosts only | — | +| rubygems | — | none at the link-writing loop | + +Go derives an owner only for module paths rooted at a known VCS host +(`github.com`, `gitlab.com`, `bitbucket.org`, `codeberg.org`, `gitea.com`, +`git.sr.ht`); a vanity module path would otherwise produce a false `unmatched`. +Rubygems stays `no_evidence` for now — owners are fetched in the critical loop, +not in the core loop that writes the link. Maven's backfill caller passes no +evidence and therefore yields `no_evidence`, not `unmatched`. + +Cargo's pipeline is set-based SQL over a dump, so it gets a SQL twin of the +matcher — `package_repo_owner_key(text)` (IMMUTABLE) and +`package_repo_owner_match(repo_owner, candidates[])` — applied as a lateral +join against the staged maintainer logins, with the repo owner carried on +`cargo_sync.repo_choice` (see +[ADR-0021](./0021-secondary-manifest-repository-signal.md)). + +## Consequences + +### Positive + +- Squatting links are separated from legitimate ones by score alone — the read + side needs no filtering, and `ORDER BY confidence DESC LIMIT 1` starts + returning the right repo for the epic's named cases. +- `no_evidence` carries a smaller penalty (−0.10) than `unmatched` (−0.25), so ecosystems with thin metadata are not scored as if they had actively mismatched the owner. +- The matcher is a pure function with standalone unit tests; the SQL twin + mirrors it explicitly rather than approximating it. + +### Negative + +- Two implementations of one rule (TypeScript and SQL) that must stay in sync. +- Heuristic matching produces false `unmatched` for legitimate publisher/owner + splits, costing those links 0.25. +- Rubygems and the Maven backfill contribute `no_evidence` until their loops + are extended, so their links carry a −0.10 they could avoid. + +### Risks + +- **The TS and SQL normalisers drift, so cargo scores differently from every + other ecosystem.** Mitigation: both were validated against the same inputs + before merge; any change to the vanity-suffix list or the prefix rule must + touch both, and the cargo integration path is the canary. +- **Vanity-suffix stripping over-matches, e.g. two distinct orgs collapsing to + the same key.** Mitigation: the 4-character floor on prefix matching, and + the penalty is a score adjustment rather than a hard rejection. +- **The `unmatched` distribution is unknown until the migration is applied.** + Mitigation: record the matched/unmatched/no_evidence baseline per ecosystem + before the rescore, so an unexpectedly large `unmatched` share is caught + before the scores reach downstream consumers. diff --git a/docs/adr/README.md b/docs/adr/README.md index 23ac0a51e3..608a3d2759 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -28,6 +28,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0019](./0019-docker-builder-runner-libc.md) | Same libc in Docker builder and runner | accepted | 2026-08-27 | | [ADR-0020](./0020-package-repo-confidence-scoring.md) | Deterministic package→repo confidence scoring | accepted | 2026-09-01 | | [ADR-0021](./0021-secondary-manifest-repository-signal.md) | Secondary manifest repository signal | accepted | 2026-09-01 | +| [ADR-0022](./0022-package-repo-ownership-evidence.md) | Ownership evidence for package→repo links | accepted | 2026-09-01 | ## Why ADRs? From 40c59375c70c1a6f522567ba77de538790c82d5f Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 17:35:22 +0100 Subject: [PATCH 5/9] fix: wire ownership-evidence matching into rubygems declared writer (CM-1394) Parse the authors field already present in the RubyGems gem response instead of adding a second fetchOwners call to the core loop, which runs over every rubygems package. Signed-off-by: Joana Maia --- services/apps/packages_worker/src/rubygems/normalize.ts | 9 +++++++++ .../packages_worker/src/rubygems/runRubyGemsCoreLoop.ts | 6 ++++++ services/apps/packages_worker/src/rubygems/types.ts | 2 ++ 3 files changed, 17 insertions(+) diff --git a/services/apps/packages_worker/src/rubygems/normalize.ts b/services/apps/packages_worker/src/rubygems/normalize.ts index 41ac62b3b3..dc53dc8f08 100644 --- a/services/apps/packages_worker/src/rubygems/normalize.ts +++ b/services/apps/packages_worker/src/rubygems/normalize.ts @@ -20,6 +20,14 @@ function cleanLicenses(raw: (string | null)[] | null | undefined): string[] | nu return cleaned && cleaned.length > 0 ? cleaned : null } +function parseAuthors(raw: string | null | undefined): string[] { + if (!raw) return [] + return raw + .split(',') + .map((a) => a.trim()) + .filter((a) => a !== '') +} + export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage { const licenses = cleanLicenses(doc.licenses) const declaredRepositoryUrl = nonEmpty(doc.source_code_uri) @@ -36,6 +44,7 @@ export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRu licensesRaw: licenses ? licenses.join(', ') : null, latestVersion: nonEmpty(doc.version), totalDownloads: doc.downloads ?? 0, + authors: parseAuthors(doc.authors), } } diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts index 721833b1ba..4b10030e44 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -13,6 +13,8 @@ import { } from '@crowd/data-access-layer' import { getServiceChildLogger } from '@crowd/logging' +import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' + import { fetchGem } from './client' import { normalizeRubyGemsPackage } from './normalize' import { BatchResult, isRubyGemsFetchError } from './types' @@ -141,6 +143,10 @@ async function processPackage( const linkChanged = await upsertPackageRepo(t, packageDbId.toString(), repoId, { source: 'declared', signal: normalized.resolvedRepo.signal, + ownershipMatch: matchOwnership({ + maintainers: normalized.authors, + repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), + }), }) linkChanged.forEach((f) => changed.add(f)) diff --git a/services/apps/packages_worker/src/rubygems/types.ts b/services/apps/packages_worker/src/rubygems/types.ts index 1bfeffc549..396a614e75 100644 --- a/services/apps/packages_worker/src/rubygems/types.ts +++ b/services/apps/packages_worker/src/rubygems/types.ts @@ -28,6 +28,7 @@ export interface RubyGemsGemResponse { bug_tracker_uri?: string | null licenses?: string[] | null downloads?: number + authors?: string | null } export interface RubyGemsVersionItem { @@ -52,6 +53,7 @@ export interface NormalizedRubyGemsPackage { licensesRaw: string | null latestVersion: string | null totalDownloads: number + authors: string[] } export interface NormalizedRubyGemsVersion { From df7a1dcd9e843860224a6d9fec28f306c81685e1 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 18:12:49 +0100 Subject: [PATCH 6/9] feat: emit ingest-time ownership-match counters across all package ecosystems (CM-1394) Signed-off-by: Joana Maia --- .../apps/packages_worker/src/cargo/enrich.ts | 16 ++++- .../apps/packages_worker/src/cargo/types.ts | 3 + .../apps/packages_worker/src/go/activities.ts | 25 +++++-- .../src/maven/runMavenEnrichmentLoop.ts | 71 +++++++++++++------ .../packages_worker/src/npm/activities.ts | 19 +++-- .../packages_worker/src/npm/upsertPackage.ts | 18 +++-- .../src/nuget/runNuGetEnrichmentLoop.ts | 47 ++++++++---- .../apps/packages_worker/src/nuget/types.ts | 3 +- .../src/packagist/__tests__/ingest.test.ts | 4 ++ .../__tests__/persistPackageInfo.test.ts | 1 + .../src/packagist/activities.ts | 23 ++++-- .../src/packagist/upsertPackageInfo.ts | 16 +++-- .../packages_worker/src/pypi/activities.ts | 18 +++-- .../packages_worker/src/pypi/upsertProject.ts | 20 ++++-- .../src/rubygems/runRubyGemsCoreLoop.ts | 47 ++++++++---- .../src/rubygems/runRubyGemsCriticalLoop.ts | 19 ++++- .../packages_worker/src/rubygems/types.ts | 3 +- .../src/utils/ownershipMatch.ts | 17 +++++ 18 files changed, 277 insertions(+), 93 deletions(-) diff --git a/services/apps/packages_worker/src/cargo/enrich.ts b/services/apps/packages_worker/src/cargo/enrich.ts index e89dee7ac4..2dd325e5f5 100644 --- a/services/apps/packages_worker/src/cargo/enrich.ts +++ b/services/apps/packages_worker/src/cargo/enrich.ts @@ -256,7 +256,7 @@ export async function enrichRepos(qx: QueryExecutor): Promise CROSS JOIN LATERAL (SELECT ${CARGO_CONFIDENCE} AS confidence) s ON CONFLICT (package_id, repo_id) DO UPDATE SET ${KEEP_HIGHEST_CONFLICT_UPDATE} - RETURNING package_id, repo_id, source, signal, confidence + RETURNING package_id, repo_id, source, signal, confidence, ownership_match ), diff AS ( SELECT ins.package_id, f.field @@ -276,7 +276,10 @@ export async function enrichRepos(qx: QueryExecutor): Promise ) SELECT (SELECT COUNT(*) FROM ins)::int AS links, - ARRAY(SELECT DISTINCT package_id::text FROM diff) AS package_ids`, + ARRAY(SELECT DISTINCT package_id::text FROM diff) AS package_ids, + (SELECT COUNT(*) FROM ins WHERE ownership_match = 'matched')::int AS declared_matched, + (SELECT COUNT(*) FROM ins WHERE ownership_match = 'unmatched')::int AS declared_unmatched, + (SELECT COUNT(*) FROM ins WHERE ownership_match = 'no_evidence')::int AS declared_no_evidence`, { source: REPO_LINK_SOURCE }, ) @@ -286,7 +289,14 @@ export async function enrichRepos(qx: QueryExecutor): Promise ] await rescorePackageReposForPackages(tx, [...new Set(allAffectedPackageIds)]) - return { repos: repoRow.repos, links: linkRow.links, pruned: pruneRow.pruned } + return { + repos: repoRow.repos, + links: linkRow.links, + pruned: pruneRow.pruned, + declared_matched: linkRow.declared_matched, + declared_unmatched: linkRow.declared_unmatched, + declared_no_evidence: linkRow.declared_no_evidence, + } }) } diff --git a/services/apps/packages_worker/src/cargo/types.ts b/services/apps/packages_worker/src/cargo/types.ts index 510ab5b08a..a227c447be 100644 --- a/services/apps/packages_worker/src/cargo/types.ts +++ b/services/apps/packages_worker/src/cargo/types.ts @@ -30,6 +30,9 @@ export interface EnrichReposResult { repos: number links: number pruned: number + declared_matched: number + declared_unmatched: number + declared_no_evidence: number } export interface EnrichMaintainersResult { diff --git a/services/apps/packages_worker/src/go/activities.ts b/services/apps/packages_worker/src/go/activities.ts index 96a5060716..25db8b891c 100644 --- a/services/apps/packages_worker/src/go/activities.ts +++ b/services/apps/packages_worker/src/go/activities.ts @@ -5,13 +5,19 @@ import { logAuditFieldChanges, upsertPackageRepo, } from '@crowd/data-access-layer/src/packages' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' import { getGoConfig } from '../config' import { getPackagesDb } from '../db' import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' -import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' +import { + bumpDeclaredOwnershipCounts, + emptyDeclaredOwnershipCounts, + matchOwnership, + repoOwnerFromCanonical, +} from '../utils/ownershipMatch' import { fetchStatus } from './pkgGoDevClient' import { fetchLatest } from './proxyClient' @@ -115,6 +121,7 @@ export async function enrichGoVersionsBatch( if (rows.length === 0) return null const { fetchTimeoutMs, proxyConcurrency } = getGoConfig() + const ownershipCounts = emptyDeclaredOwnershipCounts() const enrichOne = async (row: GoRow): Promise => { Context.current().heartbeat(row.purl) @@ -174,12 +181,15 @@ export async function enrichGoVersionsBatch( ) changedFields.push(...repoChanged) + const ownershipMatch: PackageRepoOwnershipMatch = matchOwnership({ + namespace: goModuleOwner(row.name), + repoOwner: goRepoOwner(repoToLink), + }) + bumpDeclaredOwnershipCounts(ownershipCounts, ownershipMatch) + const linkChanged = await upsertPackageRepo(t, row.id, repoId, { source: 'declared', - ownershipMatch: matchOwnership({ - namespace: goModuleOwner(row.name), - repoOwner: goRepoOwner(repoToLink), - }), + ownershipMatch, }) changedFields.push(...linkChanged) } @@ -192,7 +202,10 @@ export async function enrichGoVersionsBatch( await Promise.all(rows.slice(i, i + proxyConcurrency).map(enrichOne)) } - log.info({ count: rows.length, concurrency: proxyConcurrency }, 'Enriched go versions batch') + log.info( + { count: rows.length, concurrency: proxyConcurrency, ...ownershipCounts }, + 'Enriched go versions batch', + ) return nextCursor } diff --git a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts index 28caff0488..02ca004b83 100644 --- a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts @@ -15,11 +15,19 @@ import { upsertRepo, upsertVersionsBatch, } from '@crowd/data-access-layer' -import type { PackageRepoSignal } from '@crowd/data-access-layer/src/packages/repoConfidence' +import type { + PackageRepoOwnershipMatch, + PackageRepoSignal, +} from '@crowd/data-access-layer/src/packages/repoConfidence' import { getServiceChildLogger } from '@crowd/logging' import { getMavenConfig } from '../config' -import { OwnershipEvidence, matchOwnership } from '../utils/ownershipMatch' +import { + OwnershipEvidence, + bumpDeclaredOwnershipCounts, + emptyDeclaredOwnershipCounts, + matchOwnership, +} from '../utils/ownershipMatch' import { resolveManifestRepo } from '../utils/resolveManifestRepo' import { extractArtifact, getPomCacheStats, normalizeScmUrl } from './extract' @@ -45,12 +53,20 @@ export interface BatchResult { skipped: number error: number unchanged: number + declared_matched: number + declared_unmatched: number + declared_no_evidence: number +} + +function emptyBatchResult(): BatchResult { + return { processed: 0, skipped: 0, error: 0, unchanged: 0, ...emptyDeclaredOwnershipCounts() } } type CriticalStatus = 'processed' | 'skipped' | 'unchanged' | 'error' interface CriticalPackageResult { status: CriticalStatus + ownershipMatch: PackageRepoOwnershipMatch | null } // prettier-ignore @@ -60,13 +76,13 @@ type PackageRow = MavenPackageToSync // ─── Helpers ────────────────────────────────────────────────────────────────── // prettier-ignore -export async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed?: Set, signal: PackageRepoSignal = 'primary', evidence?: Omit): Promise { +export async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed?: Set, signal: PackageRepoSignal = 'primary', evidence?: Omit): Promise { if (!repositoryUrl) { const removedFields = await removeDeclaredPackageRepo(qx, String(packageId)) removedFields.forEach((f) => changed?.add(f)) const clearedFields = await setPackageRepositoryUrl(qx, String(packageId), null) clearedFields.forEach((f) => changed?.add(f)) - return + return null } const parsed = parseRepoUrl(repositoryUrl) if (!parsed) { @@ -74,7 +90,7 @@ export async function writeRepoLink(qx: QueryExecutor, packageId: number, reposi removedFields.forEach((f) => changed?.add(f)) const clearedFields = await setPackageRepositoryUrl(qx, String(packageId), null) clearedFields.forEach((f) => changed?.add(f)) - return + return null } const repoId = await upsertRepo(qx, { url: repositoryUrl, ...parsed }) const ownershipMatch = matchOwnership({ @@ -85,6 +101,7 @@ export async function writeRepoLink(qx: QueryExecutor, packageId: number, reposi repoChanged.forEach((f) => changed?.add(f)) const removedFields = await removeDeclaredPackageRepo(qx, String(packageId), String(repoId)) removedFields.forEach((f) => changed?.add(f)) + return ownershipMatch } // Postgres deadlock (40P01) is transient: concurrent transactions upserting the same shared @@ -141,7 +158,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF if (!groupId) { log.warn({ purl: pkg.purl }, 'Skipping: null namespace (groupId)') - return { status: 'skipped' } + return { status: 'skipped', ownershipMatch: null } } let baseUrl = resolveRegistryBaseUrl(groupId) @@ -210,14 +227,14 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF dependentReposCount: pkg.dependentReposCount, }) log.warn({ groupId, artifactId, baseUrl }, 'Not found in registry — writing minimal record') - return { status: 'skipped' } + return { status: 'skipped', ownershipMatch: null } } if (metadata.kind === 'RATE_LIMIT') { log.warn( { groupId, artifactId, status: metadata.status }, 'Rate limited — will retry next pass', ) - return { status: 'error' } + return { status: 'error', ownershipMatch: null } } throw new Error( `Transient error fetching metadata for ${groupId}:${artifactId} — ${metadata.message}`, @@ -245,7 +262,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF dependentReposCount: pkg.dependentReposCount, }) log.warn({ groupId, artifactId }, 'No release version in metadata — writing minimal record') - return { status: 'skipped' } + return { status: 'skipped', ownershipMatch: null } } // Phase 2: skip full POM extraction only if this row was already Maven-enriched — @@ -256,7 +273,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF dependentReposCount: pkg.dependentReposCount, }) log.debug({ groupId, artifactId, version }, 'Version unchanged — skipping POM extraction') - return { status: 'unchanged' } + return { status: 'unchanged', ownershipMatch: null } } // Phase 3: full POM extraction with parent-chain resolution — wrapped in a @@ -282,7 +299,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF dependentPackagesCount: pkg.dependentPackagesCount, dependentReposCount: pkg.dependentReposCount, }) - return { status: 'error' } + return { status: 'error', ownershipMatch: null } } const scmRepositoryUrl = normalizeScmUrl(result.scmUrl) @@ -291,6 +308,8 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF : resolveManifestRepo([{ field: 'url', url: result.homepageUrl, signal: 'secondary' }]) const repositoryUrl = scmRepositoryUrl ?? fallbackRepo?.repo.url ?? null + let ownershipMatch: PackageRepoOwnershipMatch | null = null + await withDeadlockRetry(() => qx.tx(async (t: QueryExecutor) => { const changed = new Set() @@ -372,7 +391,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF ) declaredClearedFields.forEach((f) => changed.add(f)) - await writeRepoLink(t, packageId, repositoryUrl, changed, fallbackRepo ? 'secondary' : 'primary', { + ownershipMatch = await writeRepoLink(t, packageId, repositoryUrl, changed, fallbackRepo ? 'secondary' : 'primary', { namespace: groupId, maintainers: allPeople.map((p) => p.username), }) @@ -394,7 +413,7 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF }), ) - return { status: 'processed' } + return { status: 'processed', ownershipMatch } } // ─── Batch processing ───────────────────────────────────────────────────────── @@ -414,7 +433,7 @@ export async function processBatch(qx: QueryExecutor, config: MavenConfig, isCri async function processPackages(qx: QueryExecutor, config: MavenConfig, packages: PackageRow[], isCritical: boolean, forceFullExtraction: boolean): Promise { const concurrency = isCritical ? config.concurrency : config.nonCriticalConcurrency - if (packages.length === 0) return { processed: 0, skipped: 0, error: 0, unchanged: 0 } + if (packages.length === 0) return emptyBatchResult() // Cluster the batch by namespace so artifacts sharing a parent POM are processed // adjacently — this is what makes the parent-POM cache effective. The criticality @@ -430,7 +449,7 @@ async function processPackages(qx: QueryExecutor, config: MavenConfig, packages: log.info({ count: packages.length, isCritical }, 'Batch started') - const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const counts = emptyBatchResult() for (let batchStart = 0; batchStart < packages.length; batchStart += concurrency) { const group = packages.slice(batchStart, batchStart + concurrency) @@ -450,6 +469,7 @@ async function processPackages(qx: QueryExecutor, config: MavenConfig, packages: const res = await processCriticalPackage(qx, pkg, forceFullExtraction) counts[res.status]++ + if (res.ownershipMatch) bumpDeclaredOwnershipCounts(counts, res.ownershipMatch) } catch (err) { const message = err instanceof Error ? err.message : String(err) log.error({ purl: pkg.purl, error: message }, 'Unexpected error processing package') @@ -477,12 +497,7 @@ async function processPackages(qx: QueryExecutor, config: MavenConfig, packages: // prettier-ignore async function runPhase(qx: QueryExecutor, config: MavenConfig, isCritical: boolean, isShuttingDown: () => boolean): Promise { const label = isCritical ? 'critical' : 'non-critical' - const total: BatchResult = { - processed: 0, - skipped: 0, - error: 0, - unchanged: 0, - } + const total: BatchResult = emptyBatchResult() let batchNum = 0 const phaseStartedAt = Date.now() @@ -503,6 +518,9 @@ async function runPhase(qx: QueryExecutor, config: MavenConfig, isCritical: bool total.skipped += result.skipped total.error += result.error total.unchanged += result.unchanged + total.declared_matched += result.declared_matched + total.declared_unmatched += result.declared_unmatched + total.declared_no_evidence += result.declared_no_evidence log.info( { @@ -512,6 +530,9 @@ async function runPhase(qx: QueryExecutor, config: MavenConfig, isCritical: bool totalSkipped: total.skipped, totalUnchanged: total.unchanged, totalErrors: total.error, + declaredMatched: total.declared_matched, + declaredUnmatched: total.declared_unmatched, + declaredNoEvidence: total.declared_no_evidence, elapsedSec: Math.round((Date.now() - phaseStartedAt) / 1000), }, 'Batch done', @@ -549,7 +570,7 @@ export async function runMavenCriticalBackfill(qx: QueryExecutor, config: MavenC */ // prettier-ignore export async function runMavenCriticalForceBackfill(qx: QueryExecutor, config: MavenConfig, isShuttingDown: () => boolean): Promise { - const total: BatchResult = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const total: BatchResult = emptyBatchResult() const startedAt = Date.now() // Cursor kept as a string: id is a Postgres bigint, and Number() coercion would // silently lose precision above 2^53, corrupting the cursor and skipping rows. @@ -573,6 +594,9 @@ export async function runMavenCriticalForceBackfill(qx: QueryExecutor, config: M total.skipped += result.skipped total.error += result.error total.unchanged += result.unchanged + total.declared_matched += result.declared_matched + total.declared_unmatched += result.declared_unmatched + total.declared_no_evidence += result.declared_no_evidence log.info( { @@ -582,6 +606,9 @@ export async function runMavenCriticalForceBackfill(qx: QueryExecutor, config: M totalSkipped: total.skipped, totalUnchanged: total.unchanged, totalErrors: total.error, + declaredMatched: total.declared_matched, + declaredUnmatched: total.declared_unmatched, + declaredNoEvidence: total.declared_no_evidence, elapsedSec: Math.round((Date.now() - startedAt) / 1000), }, 'Force batch done', diff --git a/services/apps/packages_worker/src/npm/activities.ts b/services/apps/packages_worker/src/npm/activities.ts index 66fcb6054b..c4ec366f5c 100644 --- a/services/apps/packages_worker/src/npm/activities.ts +++ b/services/apps/packages_worker/src/npm/activities.ts @@ -22,12 +22,14 @@ import { setNpmChangesLastSeq, upsertLast30dDownload, } from '@crowd/data-access-layer/src/packages' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' import { getPackagesDb } from '../db' import { proxyUrl } from '../proxies' import { isClientError } from '../utils/isClientError' +import { bumpDeclaredOwnershipCounts, emptyDeclaredOwnershipCounts } from '../utils/ownershipMatch' import { NPM_EARLIEST, computeChunks } from './downloadGaps' import { fetchChangesSince, fetchCurrentSeq } from './fetchChanges' @@ -107,17 +109,21 @@ const INGEST_4XX_BACKOFF_MS = 1000 // Fully enrich a single package. `purl` is the source-of-truth identifier from the // packages row; the npm registry name (for the HTTP fetch) is derived from it. // `dispatcher` (when present) routes the fetch through this lane's proxy IP. -async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatcher): Promise { +async function ingestOne( + qx: QueryExecutor, + purl: string, + dispatcher?: Dispatcher, +): Promise { const name = npmNameFromPurl(purl) for (let attempt = 1; attempt <= INGEST_4XX_ATTEMPTS; attempt++) { const packumentResult = await fetchPackument(name, dispatcher) if (!isFetchError(packumentResult)) { - const { changedFields } = await upsertPackage(qx, packumentResult, purl) + const { changedFields, ownershipMatch } = await upsertPackage(qx, packumentResult, purl) await logAuditFieldChanges(qx, WORKER, purl, changedFields) await markNpmPackageScanned(qx, purl, { status: 'success', attempts: attempt }) - return + return ownershipMatch } // 429 → fail the attempt, but schedule the retry past the server-stated penalty window @@ -157,6 +163,7 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche message: packumentResult.message, }) } + return null } // Number of concurrent lanes shared by all npm workers: one per configured proxy IP @@ -193,10 +200,13 @@ export async function ingestNpmPackageBatch(purls: string[], laneIndex: number): const proxy = proxyForLane(laneIndex) const dispatcher = proxy ? new ProxyAgent(proxyUrl(proxy)) : undefined + const ownershipCounts = emptyDeclaredOwnershipCounts() + try { for (const purl of pending) { await sleep(ingestSleepMs()) - await ingestOne(qx, purl, dispatcher) + const ownershipMatch = await ingestOne(qx, purl, dispatcher) + if (ownershipMatch) bumpDeclaredOwnershipCounts(ownershipCounts, ownershipMatch) } } finally { await dispatcher?.close() @@ -208,6 +218,7 @@ export async function ingestNpmPackageBatch(purls: string[], laneIndex: number): count: pending.length, skipped: purls.length - pending.length, exit: proxy?.host ?? 'direct', + ...ownershipCounts, }, 'Ingested npm package batch', ) diff --git a/services/apps/packages_worker/src/npm/upsertPackage.ts b/services/apps/packages_worker/src/npm/upsertPackage.ts index 13b91fdfa8..029d8f8a18 100644 --- a/services/apps/packages_worker/src/npm/upsertPackage.ts +++ b/services/apps/packages_worker/src/npm/upsertPackage.ts @@ -7,6 +7,7 @@ import { upsertPackageMaintainers, upsertPackageRepo, } from '@crowd/data-access-layer/src/packages' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' @@ -30,7 +31,7 @@ export async function upsertPackage( qx: QueryExecutor, packument: Packument, purl: string, -): Promise<{ purl: string; changedFields: string[] }> { +): Promise<{ purl: string; changedFields: string[]; ownershipMatch: PackageRepoOwnershipMatch | null }> { // Registry data can contain NUL bytes (e.g. mojibake descriptions) that Postgres // text columns reject; strip them before any field is persisted. stripNullBytesDeep(packument) @@ -58,6 +59,7 @@ export async function upsertPackage( const maintainers = collectMaintainers(packument) const changed = new Set() + let ownershipMatch: PackageRepoOwnershipMatch | null = null await qx.tx(async (t) => { const { id: pkgId, changedFields: pkgChanged } = await upsertNpmPackage(t, { @@ -91,14 +93,16 @@ export async function upsertPackage( ) repoChanged.forEach((f) => changed.add(f)) + ownershipMatch = matchOwnership({ + namespace, + maintainers: maintainers.filter((m) => m.role === 'maintainer').map((m) => m.username), + repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), + }) + const linkChanged = await upsertPackageRepo(t, pkgId, repoId, { source: 'declared', signal: resolvedRepo.signal, - ownershipMatch: matchOwnership({ - namespace, - maintainers: maintainers.filter((m) => m.role === 'maintainer').map((m) => m.username), - repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), - }), + ownershipMatch, }) linkChanged.forEach((f) => changed.add(f)) @@ -133,7 +137,7 @@ export async function upsertPackage( } }) - return { purl, changedFields: Array.from(changed) } + return { purl, changedFields: Array.from(changed), ownershipMatch } } function cleanKeywords(raw: unknown): string[] | null { diff --git a/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts b/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts index 40d7265f80..7e204ade47 100644 --- a/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts @@ -16,10 +16,16 @@ import { upsertNuGetVersionsBatch, upsertPackageRepo, } from '@crowd/data-access-layer' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import { getServiceChildLogger } from '@crowd/logging' import { getNuGetConfig } from '../config' -import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' +import { + bumpDeclaredOwnershipCounts, + emptyDeclaredOwnershipCounts, + matchOwnership, + repoOwnerFromCanonical, +} from '../utils/ownershipMatch' import { fetchNuspec, fetchRegistration, fetchSearch } from './client' import { normalizeNuGetPackage } from './normalize' @@ -54,12 +60,17 @@ function nugetRegistryUrl(packageId: string): string { type PackageStatus = 'processed' | 'skipped' | 'error' | 'unchanged' +interface ProcessPackageResult { + status: PackageStatus + ownershipMatch: PackageRepoOwnershipMatch | null +} + async function processPackage( qx: QueryExecutor, pkg: PackageRow, config: NuGetConfig, today: string, -): Promise { +): Promise { const packageId = pkg.name const [searchResult, registrationResult] = await Promise.all([ @@ -88,12 +99,12 @@ async function processPackage( ingestionSource: 'nuget_not_found', }) log.warn({ purl: pkg.purl }, 'Package not found on NuGet registry — writing minimal record') - return 'skipped' + return { status: 'skipped', ownershipMatch: null } } if (registrationResult.kind === 'RATE_LIMIT') { log.warn({ purl: pkg.purl }, 'Rate limited by NuGet registry — will retry next pass') await markNuGetPackageError(qx, pkg.purl) - return 'error' + return { status: 'error', ownershipMatch: null } } throw new Error( `Transient error fetching registration for ${pkg.purl}: ${registrationResult.message}`, @@ -123,6 +134,8 @@ async function processPackage( const repoUnknown = nuspecRateLimited || (searchRateLimited && normalized.resolvedRepo?.signal !== 'primary') + let ownershipMatch: PackageRepoOwnershipMatch | null = null + await withDeadlockRetry(() => qx.tx(async (t) => { const changed = new Set() @@ -173,13 +186,15 @@ async function processPackage( ) repoChanged.forEach((f) => changed.add(f)) + ownershipMatch = matchOwnership({ + maintainers: [...normalized.owners, ...normalized.authors], + repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), + }) + const linkChanged = await upsertPackageRepo(t, packageDbId.toString(), repoId, { source: 'declared', signal: normalized.resolvedRepo.signal, - ownershipMatch: matchOwnership({ - maintainers: [...normalized.owners, ...normalized.authors], - repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), - }), + ownershipMatch, }) linkChanged.forEach((f) => changed.add(f)) @@ -259,7 +274,7 @@ async function processPackage( }), ) - return 'processed' + return { status: 'processed', ownershipMatch } } export async function processBatch( @@ -272,11 +287,18 @@ export async function processBatch( isCritical: config.isCritical, }) - if (packages.length === 0) return { processed: 0, skipped: 0, error: 0, unchanged: 0 } + if (packages.length === 0) + return { processed: 0, skipped: 0, error: 0, unchanged: 0, ...emptyDeclaredOwnershipCounts() } log.info({ count: packages.length }, 'Batch started') - const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const counts: BatchResult = { + processed: 0, + skipped: 0, + error: 0, + unchanged: 0, + ...emptyDeclaredOwnershipCounts(), + } for (let batchStart = 0; batchStart < packages.length; batchStart += config.concurrency) { const group = packages.slice(batchStart, batchStart + config.concurrency) @@ -288,8 +310,9 @@ export async function processBatch( await Promise.all( group.map(async (pkg) => { try { - const status = await processPackage(qx, pkg, config, today) + const { status, ownershipMatch } = await processPackage(qx, pkg, config, today) counts[status]++ + if (ownershipMatch) bumpDeclaredOwnershipCounts(counts, ownershipMatch) } catch (err) { const message = err instanceof Error ? err.message : String(err) log.error({ purl: pkg.purl, error: message }, 'Unexpected error processing package') diff --git a/services/apps/packages_worker/src/nuget/types.ts b/services/apps/packages_worker/src/nuget/types.ts index cb279b816b..6286c94b72 100644 --- a/services/apps/packages_worker/src/nuget/types.ts +++ b/services/apps/packages_worker/src/nuget/types.ts @@ -1,3 +1,4 @@ +import { DeclaredOwnershipCounts } from '../utils/ownershipMatch' import { ResolvedManifestRepo } from '../utils/resolveManifestRepo' export interface NuGetConfig { @@ -8,7 +9,7 @@ export interface NuGetConfig { userAgent: string | undefined } -export interface BatchResult { +export interface BatchResult extends DeclaredOwnershipCounts { processed: number skipped: number error: number diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 4ba423a371..1d58e83a04 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -109,6 +109,7 @@ describe('ingestOnePackagistMetadata', () => { changedFields: ['packages.description'], packageId: '1', hasPrimaryRepo: true, + ownershipMatch: null, }) mockFetchP2.mockResolvedValue({ minifiedVersions: minified, @@ -236,6 +237,7 @@ describe('ingestOnePackagistMetadata', () => { changedFields: [], packageId: '1', hasPrimaryRepo: true, + ownershipMatch: null, }) mockFetchP2.mockResolvedValue({ kind: 'NOT_FOUND', @@ -268,6 +270,7 @@ describe('ingestOnePackagistMetadata', () => { changedFields: ['packages.description'], packageId: '1', hasPrimaryRepo: true, + ownershipMatch: null, }) mockFetchP2.mockResolvedValue({ kind: 'NOT_FOUND', @@ -295,6 +298,7 @@ describe('ingestOnePackagistMetadata', () => { changedFields: ['packages.description'], packageId: '1', hasPrimaryRepo: true, + ownershipMatch: null, }) mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index 8c556c1d66..40fd24b1aa 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -255,6 +255,7 @@ describe('persistPackagistPackageInfo', () => { changedFields: [], packageId: null, hasPrimaryRepo: true, + ownershipMatch: null, }) expect(mockRepoGet).not.toHaveBeenCalled() expect(mockMaintainers).not.toHaveBeenCalled() diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index d7731b09f6..22d5da5fa7 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -18,6 +18,7 @@ import type { PackagistMetadataCandidate, PackagistRunResult, } from '@crowd/data-access-layer/src/packages/packagistPackageState' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import { createPackagistTransitiveRun, failPackagistTransitiveRun as failRunInLedger, @@ -40,6 +41,7 @@ import { TEMPORAL_CONFIG, getTemporalClient } from '@crowd/temporal' import { getPackagesDb } from '../db' import { mapWithConcurrency } from '../utils/concurrency' import { isClientError } from '../utils/isClientError' +import { bumpDeclaredOwnershipCounts, emptyDeclaredOwnershipCounts } from '../utils/ownershipMatch' import { persistPackagist30dWindow } from './downloads' import { expandComposerMetadata } from './expandMetadata' @@ -147,7 +149,7 @@ export async function ingestOnePackagistMetadata( // This batch activity's own scheduledTimestampMs (stable across Temporal retries), // passed through to every give-up write below — see MarkMetadataScannedOptions.notBefore. scheduledAt: string, -): Promise { +): Promise { const name = packagistNameFromPurl(candidate.purl) // Phase 1: dynamic endpoint @@ -168,7 +170,7 @@ export async function ingestOnePackagistMetadata( notBefore: scheduledAt, }, ) - return + return null } const stats = normalizePackagistStats(info.value.package) @@ -176,6 +178,7 @@ export async function ingestOnePackagistMetadata( // transaction — phase 1 is committed-and-audited before the p2 fetch (which can // throw) ever runs. const phase1 = await persistPackagistPackageInfo(qx, candidate.purl, stats) + const { ownershipMatch } = phase1 // Phase 2: p2 endpoint const p2 = await fetchWithFastRetry( @@ -194,7 +197,7 @@ export async function ingestOnePackagistMetadata( bumpLastRunAt: false, notBefore: scheduledAt, }) - return + return ownershipMatch } let lastModified: string | null = null @@ -230,6 +233,8 @@ export async function ingestOnePackagistMetadata( metadataLastModified: lastModified, }, ) + + return ownershipMatch } // The monthly downloads-30d lane: dynamic fetch, one window row per purl per month. @@ -394,6 +399,8 @@ export async function ingestPackagistMetadataBatch( // Stable across every Temporal retry of this same batch — see notBefore below. const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() + const ownershipCounts = emptyDeclaredOwnershipCounts() + // The merged lane starts every ingest with a DYNAMIC-endpoint fetch, so it is // bounded by that endpoint's 10-concurrent limit — not p2's 20. Running hotter // gets connections reset by packagist.org ("fetch failed"). @@ -401,7 +408,10 @@ export async function ingestPackagistMetadataBatch( candidates, attempt, statsConcurrency(), - (candidate) => ingestOnePackagistMetadata(qx, candidate, scheduledAt), + async (candidate) => { + const ownershipMatch = await ingestOnePackagistMetadata(qx, candidate, scheduledAt) + if (ownershipMatch) bumpDeclaredOwnershipCounts(ownershipCounts, ownershipMatch) + }, (candidate, err) => markPackagistMetadataScanned( qx, @@ -422,7 +432,10 @@ export async function ingestPackagistMetadataBatch( ), ) - log.info({ count: candidates.length }, 'Ingested Packagist metadata batch') + log.info( + { count: candidates.length, ...ownershipCounts }, + 'Ingested Packagist metadata batch', + ) } export async function getPackagist30dBatch( diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 9b6c432a7f..dd7fee02de 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -8,6 +8,7 @@ import { upsertPackageMaintainers, upsertPackageRepo, } from '@crowd/data-access-layer/src/packages' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' @@ -35,6 +36,7 @@ export async function persistPackagistPackageInfo( changedFields: string[] packageId: string | null hasPrimaryRepo: boolean + ownershipMatch: PackageRepoOwnershipMatch | null }> { // Registry data can contain NUL bytes (e.g. mojibake descriptions) that Postgres // text columns reject; strip them before any field is persisted. @@ -51,6 +53,7 @@ export async function persistPackagistPackageInfo( let found = false let packageId: string | null = null const changedFields: string[] = [] + let ownershipMatch: PackageRepoOwnershipMatch | null = null await qx.tx(async (t) => { // This endpoint carries no homepage — only needed as a fallback, peek at the one @@ -86,14 +89,15 @@ export async function persistPackagistPackageInfo( // would leave a stale one dangling. if (resolvedRepo) { const repo = await getOrCreateRepoByUrl(t, resolvedRepo.repo.url, resolvedRepo.repo.host) + ownershipMatch = matchOwnership({ + namespace: stats.name.split('/')[0] || null, + maintainers: stats.maintainers.map((m) => m.username), + repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), + }) const linkChanged = await upsertPackageRepo(t, id, repo.id, { source: 'declared', signal: resolvedRepo.signal, - ownershipMatch: matchOwnership({ - namespace: stats.name.split('/')[0] || null, - maintainers: stats.maintainers.map((m) => m.username), - repoOwner: repoOwnerFromCanonical(resolvedRepo.repo), - }), + ownershipMatch, }) const removedFields = await removeDeclaredPackageRepo(t, id, repo.id) changedFields.push(...repo.changedFields, ...linkChanged, ...removedFields) @@ -119,7 +123,7 @@ export async function persistPackagistPackageInfo( await logAuditFieldChanges(t, WORKER, purl, changedFields) }) - return { found, changedFields, packageId, hasPrimaryRepo: !!primaryRepo } + return { found, changedFields, packageId, hasPrimaryRepo: !!primaryRepo, ownershipMatch } } // Phase 1 resolves the homepage-fallback repo from whatever's already stored; phase 2 diff --git a/services/apps/packages_worker/src/pypi/activities.ts b/services/apps/packages_worker/src/pypi/activities.ts index 58429d9610..8c2108b9dd 100644 --- a/services/apps/packages_worker/src/pypi/activities.ts +++ b/services/apps/packages_worker/src/pypi/activities.ts @@ -6,12 +6,14 @@ import { logAuditFieldChanges, markPypiPackageScanned, } from '@crowd/data-access-layer/src/packages' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' import { getPackagesDb } from '../db' import { proxyUrl } from '../proxies' import { isClientError } from '../utils/isClientError' +import { bumpDeclaredOwnershipCounts, emptyDeclaredOwnershipCounts } from '../utils/ownershipMatch' import { fetchProject } from './fetchProject' import { pypiNameFromPurl } from './normalize' @@ -71,17 +73,17 @@ export async function ingestOne( qx: QueryExecutor, purl: string, dispatcher?: Dispatcher, -): Promise { +): Promise { const name = pypiNameFromPurl(purl) for (let attempt = 1; attempt <= INGEST_4XX_ATTEMPTS; attempt++) { const result = await fetchProject(name, dispatcher) if (!isFetchError(result)) { - const { changedFields } = await upsertProject(qx, result, purl) + const { changedFields, ownershipMatch } = await upsertProject(qx, result, purl) await logAuditFieldChanges(qx, WORKER, purl, changedFields) await markPypiPackageScanned(qx, purl, { status: 'success', attempts: attempt }) - return + return ownershipMatch } if (!isClientError(result.statusCode, result.kind) && result.kind !== 'MALFORMED') { @@ -104,6 +106,7 @@ export async function ingestOne( message: result.message, }) } + return null } // Process purls sequentially. On a transient throw, rethrow so Temporal retries the whole @@ -166,14 +169,19 @@ export async function ingestPypiPackageBatch(purls: string[]): Promise { const attempt = Context.current().info.attempt const agents = pypiProxyPool().map((p) => new ProxyAgent(proxyUrl(p))) + const ownershipCounts = emptyDeclaredOwnershipCounts() try { await ingestPurlsWithGiveUp(qx, purls, attempt, async (purl, i) => { await sleep(ingestSleepMs()) const dispatcher = agents.length ? agents[i % agents.length] : undefined - await ingestOne(qx, purl, dispatcher) + const ownershipMatch = await ingestOne(qx, purl, dispatcher) + if (ownershipMatch) bumpDeclaredOwnershipCounts(ownershipCounts, ownershipMatch) }) } finally { await Promise.all(agents.map((a) => a.close())) } - log.info({ count: purls.length, proxied: agents.length }, 'Ingested PyPI package batch') + log.info( + { count: purls.length, proxied: agents.length, ...ownershipCounts }, + 'Ingested PyPI package batch', + ) } diff --git a/services/apps/packages_worker/src/pypi/upsertProject.ts b/services/apps/packages_worker/src/pypi/upsertProject.ts index 0175be5879..457df027b0 100644 --- a/services/apps/packages_worker/src/pypi/upsertProject.ts +++ b/services/apps/packages_worker/src/pypi/upsertProject.ts @@ -7,7 +7,10 @@ import { upsertPypiPackage, upsertPypiVersions, } from '@crowd/data-access-layer/src/packages' -import type { PackageRepoSignal } from '@crowd/data-access-layer/src/packages/repoConfidence' +import type { + PackageRepoOwnershipMatch, + PackageRepoSignal, +} from '@crowd/data-access-layer/src/packages/repoConfidence' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' @@ -28,7 +31,7 @@ export async function upsertProject( qx: QueryExecutor, project: PyPiProject, purl: string, -): Promise<{ purl: string; changedFields: string[] }> { +): Promise<{ purl: string; changedFields: string[]; ownershipMatch: PackageRepoOwnershipMatch | null }> { stripNullBytesDeep(project) const info = project.info @@ -72,6 +75,7 @@ export async function upsertProject( ) const changed = new Set() + let ownershipMatch: PackageRepoOwnershipMatch | null = null await qx.tx(async (t) => { const { id: pkgId, changedFields: pkgChanged } = await upsertPypiPackage(t, { @@ -101,13 +105,15 @@ export async function upsertProject( repo.host, ) repoChanged.forEach((f) => changed.add(f)) + ownershipMatch = matchOwnership({ + maintainers: maintainers.map((m) => m.username), + repoOwner: repoOwnerFromCanonical(repo), + }) + const linkChanged = await upsertPackageRepo(t, pkgId, repoId, { source: 'declared', signal: repoSignal, - ownershipMatch: matchOwnership({ - maintainers: maintainers.map((m) => m.username), - repoOwner: repoOwnerFromCanonical(repo), - }), + ownershipMatch, }) linkChanged.forEach((f) => changed.add(f)) @@ -134,5 +140,5 @@ export async function upsertProject( } }) - return { purl, changedFields: Array.from(changed) } + return { purl, changedFields: Array.from(changed), ownershipMatch } } diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts index 4b10030e44..bc667e93d6 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -11,9 +11,15 @@ import { upsertPackage, upsertPackageRepo, } from '@crowd/data-access-layer' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import { getServiceChildLogger } from '@crowd/logging' -import { matchOwnership, repoOwnerFromCanonical } from '../utils/ownershipMatch' +import { + bumpDeclaredOwnershipCounts, + emptyDeclaredOwnershipCounts, + matchOwnership, + repoOwnerFromCanonical, +} from '../utils/ownershipMatch' import { fetchGem } from './client' import { normalizeRubyGemsPackage } from './normalize' @@ -63,12 +69,17 @@ type PackageStatus = 'processed' | 'skipped' | 'error' | 'unchanged' export type RubyGemsCoreConfig = { batchSize: number; concurrency: number } +interface ProcessPackageResult { + status: PackageStatus + ownershipMatch: PackageRepoOwnershipMatch | null +} + async function processPackage( qx: QueryExecutor, pkg: RubyGemsPackageToSync, today: string, signal?: AbortSignal, -): Promise { +): Promise { const gemResult = await fetchGem(pkg.name, signal) if (isRubyGemsFetchError(gemResult)) { @@ -91,17 +102,19 @@ async function processPackage( { purl: pkg.purl }, 'Package not found on RubyGems registry — writing minimal record', ) - return 'skipped' + return { status: 'skipped', ownershipMatch: null } } if (gemResult.kind === 'RATE_LIMIT') { log.warn({ purl: pkg.purl }, 'Rate limited by RubyGems registry — will retry next pass') - return 'error' + return { status: 'error', ownershipMatch: null } } throw new Error(`Transient error fetching ${pkg.purl}: ${gemResult.message}`) } const normalized = normalizeRubyGemsPackage(gemResult) + let ownershipMatch: PackageRepoOwnershipMatch | null = null + await withDeadlockRetry(() => qx.tx(async (t) => { const changed = new Set() @@ -140,13 +153,15 @@ async function processPackage( ) repoChanged.forEach((f) => changed.add(f)) + ownershipMatch = matchOwnership({ + maintainers: normalized.authors, + repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), + }) + const linkChanged = await upsertPackageRepo(t, packageDbId.toString(), repoId, { source: 'declared', signal: normalized.resolvedRepo.signal, - ownershipMatch: matchOwnership({ - maintainers: normalized.authors, - repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), - }), + ownershipMatch, }) linkChanged.forEach((f) => changed.add(f)) @@ -175,7 +190,7 @@ async function processPackage( }), ) - return 'processed' + return { status: 'processed', ownershipMatch } } export async function processBatch( @@ -186,11 +201,18 @@ export async function processBatch( ): Promise { const packages = await listRubyGemsPackagesToSync(qx, { limit: config.batchSize }) - if (packages.length === 0) return { processed: 0, skipped: 0, error: 0, unchanged: 0 } + if (packages.length === 0) + return { processed: 0, skipped: 0, error: 0, unchanged: 0, ...emptyDeclaredOwnershipCounts() } log.info({ count: packages.length }, 'Batch started') - const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const counts: BatchResult = { + processed: 0, + skipped: 0, + error: 0, + unchanged: 0, + ...emptyDeclaredOwnershipCounts(), + } for (let batchStart = 0; batchStart < packages.length; batchStart += config.concurrency) { signal?.throwIfAborted() @@ -199,8 +221,9 @@ export async function processBatch( await Promise.all( group.map(async (pkg) => { try { - const status = await processPackage(qx, pkg, today, signal) + const { status, ownershipMatch } = await processPackage(qx, pkg, today, signal) counts[status]++ + if (ownershipMatch) bumpDeclaredOwnershipCounts(counts, ownershipMatch) } catch (err) { signal?.throwIfAborted() const message = err instanceof Error ? err.message : String(err) diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts index 9c7c877913..c193e1650c 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts @@ -11,6 +11,8 @@ import { } from '@crowd/data-access-layer' import { getServiceChildLogger } from '@crowd/logging' +import { emptyDeclaredOwnershipCounts } from '../utils/ownershipMatch' + import { fetchOwners, fetchVersions } from './client' import { normalizeRubyGemsOwners, @@ -156,12 +158,25 @@ export async function processBatch( }) if (packages.length === 0) { - return { processed: 0, skipped: 0, error: 0, unchanged: 0, lastId: null } + return { + processed: 0, + skipped: 0, + error: 0, + unchanged: 0, + ...emptyDeclaredOwnershipCounts(), + lastId: null, + } } log.info({ count: packages.length, afterId }, 'Critical batch started') - const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const counts: BatchResult = { + processed: 0, + skipped: 0, + error: 0, + unchanged: 0, + ...emptyDeclaredOwnershipCounts(), + } for (let batchStart = 0; batchStart < packages.length; batchStart += config.concurrency) { signal?.throwIfAborted() diff --git a/services/apps/packages_worker/src/rubygems/types.ts b/services/apps/packages_worker/src/rubygems/types.ts index 396a614e75..11d0037dc6 100644 --- a/services/apps/packages_worker/src/rubygems/types.ts +++ b/services/apps/packages_worker/src/rubygems/types.ts @@ -1,6 +1,7 @@ +import { DeclaredOwnershipCounts } from '../utils/ownershipMatch' import { ResolvedManifestRepo } from '../utils/resolveManifestRepo' -export interface BatchResult { +export interface BatchResult extends DeclaredOwnershipCounts { processed: number skipped: number error: number diff --git a/services/apps/packages_worker/src/utils/ownershipMatch.ts b/services/apps/packages_worker/src/utils/ownershipMatch.ts index cfa769b9f2..43089cb4a3 100644 --- a/services/apps/packages_worker/src/utils/ownershipMatch.ts +++ b/services/apps/packages_worker/src/utils/ownershipMatch.ts @@ -10,6 +10,23 @@ export interface OwnershipEvidence { repoOwner: string | null } +export interface DeclaredOwnershipCounts { + declared_matched: number + declared_unmatched: number + declared_no_evidence: number +} + +export function emptyDeclaredOwnershipCounts(): DeclaredOwnershipCounts { + return { declared_matched: 0, declared_unmatched: 0, declared_no_evidence: 0 } +} + +export function bumpDeclaredOwnershipCounts( + counts: DeclaredOwnershipCounts, + match: PackageRepoOwnershipMatch, +): void { + counts[`declared_${match}`]++ +} + const VANITY_SUFFIXES = ['-ai', '-io', '-team', '-labs', '-oss', '-dev'] function normalizeIdentity(raw: string): string { From babbf132bb1e91374eeb6560582d6639bb898377 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 8 Sep 2026 18:58:47 +0100 Subject: [PATCH 7/9] fix: address bot review findings on ownership-evidence PR (CM-1394) Fixes the DROP FUNCTION dependency failure (9-arg compat overload from V1788307300 still referenced the 10-arg signature), restores a rolling-deploy compat overload for the dropped 10-arg signature, restores the verified_at monotonic bump in both rescore paths so Tinybird's ReplacingMergeTree doesn't drop concurrent updates, adds ownership_match to the Tinybird packageRepos datasource and to the upsertPackageRepo audit diff, stops RubyGems from feeding free-text author names into ownership matching (ADR-0022 says that loop stays no_evidence), and fixes a stale migration reference in a comment. Also fixes prettier formatting drift in npm/pypi/packagist writers and a forbidden non-null assertion in ownershipMatch.ts that were failing the lint-format-services CI check. Signed-off-by: Joana Maia --- ...8393601__no_evidence_ownership_penalty.sql | 32 ++++++++++++++++++- .../src/deps-dev/workflows/ingestRepos.ts | 3 +- .../packages_worker/src/npm/upsertPackage.ts | 6 +++- .../src/packagist/__tests__/ingest.test.ts | 1 + .../src/packagist/activities.ts | 7 ++-- .../packages_worker/src/pypi/upsertProject.ts | 6 +++- .../src/rubygems/runRubyGemsCoreLoop.ts | 4 ++- .../src/utils/ownershipMatch.ts | 4 +-- .../src/osspckgs/sqlFragments.ts | 2 +- .../data-access-layer/src/packages/repos.ts | 6 ++-- .../datasources/packageRepos.datasource | 2 ++ 11 files changed, 58 insertions(+), 15 deletions(-) diff --git a/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql index 5997b5e47c..7267fd217f 100644 --- a/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql +++ b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql @@ -7,6 +7,11 @@ -- Adding a parameter changes the signature, so the V1788307300 function is dropped -- rather than replaced — CREATE OR REPLACE would leave both overloads callable. +-- The 9-arg compat overload from V1788307300 depends on the 10-arg one below, so it +-- must go first or the DROP FUNCTION on the 10-arg signature fails on that dependency. +DROP FUNCTION IF EXISTS package_repo_confidence( + text, text, text, bool, bool, bool, text, bool, bigint +); DROP FUNCTION IF EXISTS package_repo_confidence( text, text, text, text, bool, bool, bool, text, bool, bigint ); @@ -100,6 +105,30 @@ BEGIN END; $$; +-- Compat overload for callers still on the pre-ownership-match signature during a rolling +-- deploy; delegates to the widened function as 'no_evidence' (the safest default — it's the +-- same penalty already applied to deps.dev links with no ownership evidence). Drop in a +-- later cleanup migration once all writers emit the 11-arg call. +CREATE OR REPLACE FUNCTION package_repo_confidence( + p_source text, + p_ecosystem text, + p_signal text, + p_provenance text, + p_archived bool, + p_is_fork bool, + p_disabled bool, + p_host text, + p_competing_github bool, + p_repo_id bigint +) +RETURNS numeric(12, 9) +LANGUAGE sql IMMUTABLE AS $$ + SELECT package_repo_confidence( + p_source, p_ecosystem, p_signal, 'no_evidence', p_provenance, + p_archived, p_is_fork, p_disabled, p_host, p_competing_github, p_repo_id + ) +$$; + -- Replaced only to pass cur.ownership_match through to the widened scoring function; -- the chunking, locking and keyset paging are unchanged from V1788307300. CREATE OR REPLACE PROCEDURE rescore_package_repo_confidence( @@ -140,7 +169,8 @@ BEGIN ), updated AS ( UPDATE package_repos pr - SET confidence = s.confidence, verified_at = NOW() + SET confidence = s.confidence, + verified_at = GREATEST(clock_timestamp(), cur.verified_at + interval '1 millisecond') FROM batch b JOIN package_repos cur ON cur.id = b.id JOIN packages p ON p.id = cur.package_id diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts index b8d468202f..2e7f44ad95 100644 --- a/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts +++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts @@ -107,7 +107,8 @@ const PKGREPOS_PG_COLUMNS = ['purl', 'canonical_url', 'provenance'] // instead via competingGithubRepoExpr which reads the live package_repos table. const PKGREPOS_RESCORE_SQL = ` UPDATE package_repos pr - SET confidence = s.confidence + SET confidence = s.confidence, + verified_at = GREATEST(clock_timestamp(), pr.verified_at + interval '1 millisecond') FROM packages p, repos r, LATERAL ( SELECT ${packageRepoConfidenceCall('p', 'r', claimFromRow('pr'), competingGithubRepoExpr('p.id', 'r.id'))} AS confidence diff --git a/services/apps/packages_worker/src/npm/upsertPackage.ts b/services/apps/packages_worker/src/npm/upsertPackage.ts index 029d8f8a18..4a5b297a00 100644 --- a/services/apps/packages_worker/src/npm/upsertPackage.ts +++ b/services/apps/packages_worker/src/npm/upsertPackage.ts @@ -31,7 +31,11 @@ export async function upsertPackage( qx: QueryExecutor, packument: Packument, purl: string, -): Promise<{ purl: string; changedFields: string[]; ownershipMatch: PackageRepoOwnershipMatch | null }> { +): Promise<{ + purl: string + changedFields: string[] + ownershipMatch: PackageRepoOwnershipMatch | null +}> { // Registry data can contain NUL bytes (e.g. mojibake descriptions) that Postgres // text columns reject; strip them before any field is persisted. stripNullBytesDeep(packument) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 1d58e83a04..3103554691 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -151,6 +151,7 @@ describe('ingestOnePackagistMetadata', () => { changedFields: ['packages.description'], packageId: '1', hasPrimaryRepo: false, + ownershipMatch: null, }) mockPersistMetadata.mockResolvedValue({ found: true, diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 22d5da5fa7..a67180a4b3 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -18,7 +18,6 @@ import type { PackagistMetadataCandidate, PackagistRunResult, } from '@crowd/data-access-layer/src/packages/packagistPackageState' -import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import { createPackagistTransitiveRun, failPackagistTransitiveRun as failRunInLedger, @@ -27,6 +26,7 @@ import { hasRecentDonePackagistTransitiveRun, markPackagistTransitiveRunMerging, } from '@crowd/data-access-layer/src/packages/packagistTransitiveRuns' +import type { PackageRepoOwnershipMatch } from '@crowd/data-access-layer/src/packages/repoConfidence' import { EmptyPackagistTransitiveCountsError, computePackagistTransitiveCounts, @@ -432,10 +432,7 @@ export async function ingestPackagistMetadataBatch( ), ) - log.info( - { count: candidates.length, ...ownershipCounts }, - 'Ingested Packagist metadata batch', - ) + log.info({ count: candidates.length, ...ownershipCounts }, 'Ingested Packagist metadata batch') } export async function getPackagist30dBatch( diff --git a/services/apps/packages_worker/src/pypi/upsertProject.ts b/services/apps/packages_worker/src/pypi/upsertProject.ts index 457df027b0..ec47ae1fe9 100644 --- a/services/apps/packages_worker/src/pypi/upsertProject.ts +++ b/services/apps/packages_worker/src/pypi/upsertProject.ts @@ -31,7 +31,11 @@ export async function upsertProject( qx: QueryExecutor, project: PyPiProject, purl: string, -): Promise<{ purl: string; changedFields: string[]; ownershipMatch: PackageRepoOwnershipMatch | null }> { +): Promise<{ + purl: string + changedFields: string[] + ownershipMatch: PackageRepoOwnershipMatch | null +}> { stripNullBytesDeep(project) const info = project.info diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts index bc667e93d6..d046502377 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -153,8 +153,10 @@ async function processPackage( ) repoChanged.forEach((f) => changed.add(f)) + // Gem authors are free-text display names, not GitHub identity handles, so they + // can't be used as maintainer evidence — per ADR-0022, this loop stays no_evidence + // until owners are fetched in the critical loop. ownershipMatch = matchOwnership({ - maintainers: normalized.authors, repoOwner: repoOwnerFromCanonical(normalized.resolvedRepo.repo), }) diff --git a/services/apps/packages_worker/src/utils/ownershipMatch.ts b/services/apps/packages_worker/src/utils/ownershipMatch.ts index 43089cb4a3..096dbe9730 100644 --- a/services/apps/packages_worker/src/utils/ownershipMatch.ts +++ b/services/apps/packages_worker/src/utils/ownershipMatch.ts @@ -115,8 +115,8 @@ export function matchOwnership(evidence: OwnershipEvidence): PackageRepoOwnershi const candidates = [ ...(evidence.namespace ? namespaceCandidates(evidence.namespace) : []), ...(evidence.maintainers ?? []) - .filter((m) => m && !/\S@\S/.test(m)) - .map((m) => normalizeIdentity(m!)), + .filter((m): m is string => Boolean(m) && !/\S@\S/.test(m as string)) + .map((m) => normalizeIdentity(m)), ].filter(Boolean) if (candidates.length === 0) return 'no_evidence' diff --git a/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts b/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts index f45d84ac0b..1c4b991893 100644 --- a/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts +++ b/services/libs/data-access-layer/src/osspckgs/sqlFragments.ts @@ -18,7 +18,7 @@ export const STEWARD_DISPLAY_NAME_METADATA = `CASE ELSE sa.metadata END` -// Ranking of a package's repo links. The uniqueness offset in confidence (V1788393601) +// Ranking of a package's repo links. The uniqueness offset in confidence (V1788307200) // is not injective, so repo_id is what makes the order total. Mirrored in ossPackages_enriched. export function bestRepoLinkOrderBy(alias: string): string { return `ORDER BY ${alias}.confidence DESC, ${alias}.repo_id DESC` diff --git a/services/libs/data-access-layer/src/packages/repos.ts b/services/libs/data-access-layer/src/packages/repos.ts index 15b9d72b4c..af49fd39e1 100644 --- a/services/libs/data-access-layer/src/packages/repos.ts +++ b/services/libs/data-access-layer/src/packages/repos.ts @@ -96,7 +96,7 @@ export async function upsertPackageRepo( const row: { changed_fields: string[] } | null = await qx.selectOneOrNone( `WITH old AS ( - SELECT source, signal, confidence FROM package_repos + SELECT source, signal, ownership_match, confidence FROM package_repos WHERE package_id = $(packageId)::bigint AND repo_id = $(repoId)::bigint ), scored AS ( @@ -115,7 +115,7 @@ export async function upsertPackageRepo( FROM scored ON CONFLICT (package_id, repo_id) DO UPDATE SET ${KEEP_HIGHEST_CONFLICT_UPDATE} - RETURNING source, signal, confidence + RETURNING source, signal, ownership_match, confidence ) SELECT array_remove(ARRAY[ CASE WHEN o.source IS NULL THEN 'package_repos.repo_id' END, @@ -123,6 +123,8 @@ export async function upsertPackageRepo( OR o.source IS DISTINCT FROM ins.source THEN 'package_repos.source' END, CASE WHEN o.source IS NULL OR o.signal IS DISTINCT FROM ins.signal THEN 'package_repos.signal' END, + CASE WHEN o.source IS NULL + OR o.ownership_match IS DISTINCT FROM ins.ownership_match THEN 'package_repos.ownership_match' END, CASE WHEN o.source IS NULL OR o.confidence IS DISTINCT FROM ins.confidence THEN 'package_repos.confidence' END ], NULL) AS changed_fields diff --git a/services/libs/tinybird/datasources/packageRepos.datasource b/services/libs/tinybird/datasources/packageRepos.datasource index e269b31208..06a4d2de1d 100644 --- a/services/libs/tinybird/datasources/packageRepos.datasource +++ b/services/libs/tinybird/datasources/packageRepos.datasource @@ -7,6 +7,7 @@ DESCRIPTION > - `repoId` links to the associated repos row. - `source` identifies how the link was established: 'declared', 'deps_dev', 'heuristic', or 'manual'. - `signal` is 'primary' or 'secondary' — whether a 'declared' link came from the ecosystem's dedicated repository field or a lower-trust fallback (homepage/bug tracker). + - `ownershipMatch` is 'matched', 'unmatched', or 'no_evidence' — whether the declared namespace/maintainer identity matches the repo owner. - `confidence` is a 0–1 score produced by the package_repo_confidence() function in packages-db, with nine decimal places; the low-order digits are a per-row uniqueness offset, not precision. - `provenance` is the deps.dev RelationProvenance for source='deps_dev' rows and empty for every other source. - `verifiedAt` is when the link was last confirmed or upserted — serves as the updated_at watermark for sync. @@ -18,6 +19,7 @@ SCHEMA > `repoId` UInt64 `json:$.record.repo_id`, `source` String `json:$.record.source`, `signal` String `json:$.record.signal` DEFAULT 'primary', + `ownershipMatch` String `json:$.record.ownership_match` DEFAULT 'no_evidence', `confidence` String `json:$.record.confidence`, `provenance` String `json:$.record.provenance` DEFAULT '', `verifiedAt` DateTime64(3) `json:$.record.verified_at`, From 29e808d02d61aa588fca7869ee2ca4288d91ec11 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 9 Sep 2026 10:24:26 +0100 Subject: [PATCH 8/9] Update test description for packageRepoLinkClaimParams Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Joana Maia --- .../libs/data-access-layer/src/packages/repoConfidence.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/data-access-layer/src/packages/repoConfidence.test.ts b/services/libs/data-access-layer/src/packages/repoConfidence.test.ts index ff32e34265..5f68c6f95d 100644 --- a/services/libs/data-access-layer/src/packages/repoConfidence.test.ts +++ b/services/libs/data-access-layer/src/packages/repoConfidence.test.ts @@ -24,7 +24,7 @@ describe('packageRepoConfidenceLabel', () => { }) describe('packageRepoLinkClaimParams', () => { - it('defaults the signals CM-1393 and CM-1394 have not started writing yet', () => { + it('defaults missing claim fields (signal, ownershipMatch, provenance)', () => { expect(packageRepoLinkClaimParams({ source: 'declared' })).toEqual({ source: 'declared', signal: 'primary', From 01d6e40949cdf2c8d5b02aeb9e2fdc8c1b0a70cc Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 9 Sep 2026 11:25:00 +0100 Subject: [PATCH 9/9] fix: correct misleading compat-overload comment in ownership-penalty migration (CM-1394) The comment claimed the no_evidence default for pre-ownership-match callers matched a penalty already applied to deps_dev links. The ownership penalty only applies when p_source = 'declared'; deps_dev is unaffected either way. Signed-off-by: Joana Maia --- .../V1788393601__no_evidence_ownership_penalty.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql index 7267fd217f..d77b246c12 100644 --- a/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql +++ b/backend/src/osspckgs/migrations/V1788393601__no_evidence_ownership_penalty.sql @@ -106,9 +106,9 @@ END; $$; -- Compat overload for callers still on the pre-ownership-match signature during a rolling --- deploy; delegates to the widened function as 'no_evidence' (the safest default — it's the --- same penalty already applied to deps.dev links with no ownership evidence). Drop in a --- later cleanup migration once all writers emit the 11-arg call. +-- deploy; delegates to the widened function as 'no_evidence', the safest default since the +-- ownership penalty only applies to p_source = 'declared' — every other source ignores it. +-- Drop in a later cleanup migration once all writers emit the 11-arg call. CREATE OR REPLACE FUNCTION package_repo_confidence( p_source text, p_ecosystem text,