diff --git a/pyproject.toml b/pyproject.toml index 609daa2..43db636 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "socketdev" -version = "3.3.0" +version = "3.4.0" requires-python = ">= 3.9" dependencies = [ 'requests', diff --git a/socketdev/core/dedupe.py b/socketdev/core/dedupe.py index b3b29da..2e6e5c4 100644 --- a/socketdev/core/dedupe.py +++ b/socketdev/core/dedupe.py @@ -11,8 +11,8 @@ def normalize_file_path(path: str) -> str: @staticmethod def alert_key(alert: dict) -> tuple: return ( - alert["type"], - alert["severity"], + alert.get("type"), + alert.get("severity"), alert.get("category"), Dedupe.normalize_file_path(alert.get("file")), alert.get("start"), @@ -23,8 +23,8 @@ def alert_key(alert: dict) -> tuple: def consolidate_and_merge_alerts(package_group: List[Dict[str, Any]]) -> Dict[str, Any]: def alert_identity(alert: dict) -> tuple: return ( - alert["type"], - alert["severity"], + alert.get("type"), + alert.get("severity"), alert.get("category"), Dedupe.normalize_file_path(alert.get("file")), alert.get("start"), @@ -41,14 +41,17 @@ def alert_identity(alert: dict) -> tuple: identity = alert_identity(alert) if identity not in alert_map: - # Build alert dict with only fields that exist in the original alert + # Build alert dict with only fields that exist in the original alert. + # Use .get() for key/type/severity/action so synthetic status rows + # (e.g. pendingScan/notFound), which are built server-side from a + # minimal {type, key} base, don't raise KeyError here. consolidated_alert = { - "key": alert["key"], # keep the first key seen - "type": alert["type"], - "severity": alert["severity"], + "key": alert.get("key"), # keep the first key seen + "type": alert.get("type"), + "severity": alert.get("severity"), "releases": [release], "props": alert.get("props", []), - "action": alert["action"] + "action": alert.get("action") } # Only include optional fields if they exist in the original alert diff --git a/socketdev/exceptions.py b/socketdev/exceptions.py index 980aaf9..737831d 100644 --- a/socketdev/exceptions.py +++ b/socketdev/exceptions.py @@ -78,3 +78,22 @@ class APIBadGateway(APIFailure): def __init__(self, *args): super().__init__(*args, status_code=502) + + +class APIPartialResponse(APIFailure): + """Raised by ``purl.post(strict=True)`` when the batch response omits requested inputs. + + The batch purl API is fail-open: input purls whose resolution/analysis has not + completed are silently dropped from the response unless the caller opts in via + ``alerts=True`` (synthetic ``pendingScan``/``notFound`` rows) or ``poll=True`` (a + bounded fail-closed wait). ``strict=True`` turns that silent omission into this + explicit error so callers get a first-class "partial batch" signal without having + to diff the response themselves. + + The ``missing`` attribute holds the requested purls that were absent from the + response (the HTTP call itself succeeded, so there is no status code). + """ + + def __init__(self, *args, missing=None): + super().__init__(*args) + self.missing = list(missing or []) diff --git a/socketdev/purl/__init__.py b/socketdev/purl/__init__.py index 50118ed..534a171 100644 --- a/socketdev/purl/__init__.py +++ b/socketdev/purl/__init__.py @@ -1,7 +1,9 @@ import json import urllib.parse import warnings +from typing import Optional from socketdev.log import log +from socketdev.exceptions import APIPartialResponse from ..core.dedupe import Dedupe @@ -14,8 +16,55 @@ def post( license: str = "false", components: list = None, org_slug: str = None, + poll: Optional[bool] = None, + timeout_sec: Optional[int] = None, + alerts: Optional[bool] = None, + purl_errors: Optional[bool] = None, + strict: bool = False, **kwargs, ) -> list: + """POST a batch of purls to the Socket batch purl endpoint and return deduped rows. + + The batch purl API (``POST /v0/purl`` and ``POST /v0/orgs/{slug}/purl``) defaults + to **fail-open**: any input purl whose resolution/analysis has not finished is + **silently omitted** from the response. A naive caller therefore cannot tell + "this version is clean" apart from "this version was dropped from the response". + The parameters below opt into the server behaviors that make omissions visible. + + Args: + license: ``"true"``/``"false"`` — request license information (stringly-typed + to match the query param the API expects). + components: list of component dicts to score, e.g. ``[{"purl": "pkg:npm/lodash@4.18.1"}]``. + org_slug: organization slug. When provided, routes to the org-scoped endpoint + ``POST /v0/orgs/{org_slug}/purl``; otherwise the deprecated ``POST /v0/purl``. + poll: opt into a fail-closed bounded wait for pending analysis (``poll=True`` → + ``poll=true`` query param). ``None`` omits the param (server default). + timeout_sec: bound in seconds for the ``poll`` wait (``→ timeoutSec``). The + server may cap this via a feature flag. ``None`` omits the param. + alerts: when ``True`` (``→ alerts=true``), the server emits synthetic + ``pendingScan``/``notFound`` status rows instead of silently omitting + unresolved inputs, so callers can distinguish "no data yet" from "clean". + purl_errors: when ``True`` (``→ purlErrors``), the server includes per-purl + error rows for malformed/unresolvable inputs. ``None`` omits the param. + strict: client-side guard. When ``True``, compares the ``purl`` of each + requested component against the ``inputPurl``/``purl`` of the returned + rows and raises :class:`~socketdev.exceptions.APIPartialResponse` (with a + ``missing`` list) if any requested purl is absent from the response. This + surfaces partial batches even without ``alerts=True``. Only components that + carry a ``purl`` string are checked. + **kwargs: forwarded verbatim into the query string (back-compat passthrough for + any params not yet promoted to first-class arguments). + + Returns: + A deduped list of result rows. When ``alerts=True``, unresolved inputs appear + as synthetic rows carrying ``pendingScan``/``notFound`` alerts rather than being + omitted. On a non-200 response, logs the error and returns ``[]`` (callers that + need to fail closed should treat ``[]`` as an error). + + Raises: + APIPartialResponse: if ``strict=True`` and one or more requested component purls + are missing from the response. + """ if org_slug is None: warnings.warn( "Calling purl.post() without org_slug uses the deprecated POST /v0/purl endpoint. " @@ -31,25 +80,83 @@ def post( query_args = { "license": license, } + # Promote the typed params into query args only when explicitly set, so existing + # callers keep the server's fail-open default (None => omit the param entirely). + if poll is not None: + query_args["poll"] = "true" if poll else "false" + if timeout_sec is not None: + query_args["timeoutSec"] = str(timeout_sec) + if alerts is not None: + query_args["alerts"] = "true" if alerts else "false" + if purl_errors is not None: + query_args["purlErrors"] = "true" if purl_errors else "false" if kwargs: query_args.update(kwargs) params = urllib.parse.urlencode(query_args) path += params response = self.api.do_request(path=path, payload=purls, method="POST") if response.status_code == 200: - purl = [] + artifact_rows = [] + stream_records = [] result = response.text result = result.strip('"').strip() for line in result.split("\n"): if line and line != '"': try: item = json.loads(line) - purl.append(item) + if isinstance(item, dict) and item.get("_type") in { + "purlError", + "summary", + }: + stream_records.append(item) + else: + artifact_rows.append(item) except json.JSONDecodeError: continue - purl_deduped = Dedupe.dedupe(purl, batched=True) + purl_deduped = Dedupe.dedupe(artifact_rows, batched=True) + purl_deduped.extend(stream_records) + if strict: + self._raise_on_missing(components, purl_deduped) return purl_deduped log.error(f"Error posting {components} to the Purl API: {response.status_code}") log.error(response.text) return [] + + @staticmethod + def _raise_on_missing(components: list, results: list) -> None: + """Raise APIPartialResponse if any requested component purl is absent from results. + + Only components exposing a ``purl`` string are checked; the batch API echoes the + request identifier back as ``inputPurl`` (falling back to ``purl``), including + under ``value`` for typed ``purlError`` stream records. + """ + requested = [ + c["purl"] + for c in components + if isinstance(c, dict) and isinstance(c.get("purl"), str) + ] + if not requested: + return + returned = set() + for row in results: + if not isinstance(row, dict): + continue + for field in ("inputPurl", "purl"): + value = row.get(field) + if isinstance(value, str): + returned.add(value) + record_value = row.get("value") + if isinstance(record_value, dict): + input_purl = record_value.get("inputPurl") + if isinstance(input_purl, str): + returned.add(input_purl) + missing = [purl for purl in requested if purl not in returned] + if missing: + raise APIPartialResponse( + "purl.post(strict=True): the batch response omitted " + f"{len(missing)} of {len(requested)} requested purls " + "(fail-open: unresolved inputs are dropped unless alerts=True/poll=True): " + f"{missing}", + missing=missing, + ) diff --git a/socketdev/version.py b/socketdev/version.py index 88c513e..903a158 100644 --- a/socketdev/version.py +++ b/socketdev/version.py @@ -1 +1 @@ -__version__ = "3.3.0" +__version__ = "3.4.0" diff --git a/tests/unit/test_all_endpoints_unit.py b/tests/unit/test_all_endpoints_unit.py index 64895ee..d263609 100644 --- a/tests/unit/test_all_endpoints_unit.py +++ b/tests/unit/test_all_endpoints_unit.py @@ -401,6 +401,133 @@ def test_purl_post_unit_legacy_path(self): self.assertIn("/purl", call_args[0][1]) self.assertNotIn("/orgs/", call_args[0][1]) + def _mock_purl_ndjson(self, ndjson): + """Mock a 200 NDJSON purl response and return the mock.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'content-type': 'application/x-ndjson'} + mock_response.text = ndjson + self.mock_requests.request.return_value = mock_response + return mock_response + + def test_purl_post_first_class_params_query_string(self): + """poll/timeout_sec/alerts/purl_errors map to the expected query params.""" + self._mock_purl_ndjson( + '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", ' + '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}' + ) + + self.sdk.purl.post( + license="false", + components=[{"purl": "pkg:npm/lodash@4.18.1"}], + org_slug="test-org", + poll=True, + timeout_sec=120, + alerts=True, + purl_errors=False, + ) + + url = self.mock_requests.request.call_args[0][1] + self.assertIn("poll=true", url) + self.assertIn("timeoutSec=120", url) + self.assertIn("alerts=true", url) + self.assertIn("purlErrors=false", url) + + def test_purl_post_omits_unset_params(self): + """None-valued typed params are omitted so the API's fail-open default is preserved.""" + self._mock_purl_ndjson( + '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", ' + '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}' + ) + + self.sdk.purl.post(components=[{"purl": "pkg:npm/lodash@4.18.1"}], org_slug="test-org") + + url = self.mock_requests.request.call_args[0][1] + self.assertNotIn("poll=", url) + self.assertNotIn("timeoutSec=", url) + self.assertNotIn("alerts=", url) + self.assertNotIn("purlErrors=", url) + + def test_purl_post_synthetic_pending_scan_row(self): + """A synthetic pendingScan row (no severity/action) parses without raising KeyError.""" + # Synthetic status alerts are built server-side from a minimal {type, key} base. + self._mock_purl_ndjson( + '{"inputPurl": "pkg:npm/newpkg@0.0.1", "purl": "pkg:npm/newpkg@0.0.1", ' + '"type": "npm", "name": "newpkg", "version": "0.0.1", ' + '"alerts": [{"type": "pendingScan", "key": "abc123"}]}' + ) + + result = self.sdk.purl.post( + components=[{"purl": "pkg:npm/newpkg@0.0.1"}], + org_slug="test-org", + alerts=True, + ) + + self.assertEqual(len(result), 1) + alert = result[0]["alerts"][0] + self.assertEqual(alert["type"], "pendingScan") + self.assertEqual(alert["key"], "abc123") + # Missing fields are surfaced as None rather than raising. + self.assertIsNone(alert["severity"]) + self.assertIsNone(alert["action"]) + + def test_purl_post_preserves_purl_error_record(self): + """purlError stream records bypass artifact deduplication.""" + error_row = { + "_type": "purlError", + "value": { + "error": "package_not_found", + "inputPurl": "pkg:npm/missing@1.0.0", + }, + } + self._mock_purl_ndjson(json.dumps(error_row)) + + result = self.sdk.purl.post( + components=[{"purl": "pkg:npm/missing@1.0.0"}], + org_slug="test-org", + purl_errors=True, + strict=True, + ) + + self.assertEqual(result, [error_row]) + url = self.mock_requests.request.call_args[0][1] + self.assertIn("purlErrors=true", url) + + def test_purl_post_strict_raises_on_missing(self): + """strict=True raises APIPartialResponse listing purls dropped from the response.""" + from socketdev.exceptions import APIPartialResponse + + # Requested two purls; the fail-open API only returned one. + self._mock_purl_ndjson( + '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", ' + '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}' + ) + + with self.assertRaises(APIPartialResponse) as ctx: + self.sdk.purl.post( + components=[ + {"purl": "pkg:npm/lodash@4.18.1"}, + {"purl": "pkg:npm/dropped@0.0.1"}, + ], + org_slug="test-org", + strict=True, + ) + self.assertEqual(ctx.exception.missing, ["pkg:npm/dropped@0.0.1"]) + + def test_purl_post_strict_passes_when_complete(self): + """strict=True returns normally when every requested purl is present.""" + self._mock_purl_ndjson( + '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", ' + '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}' + ) + + result = self.sdk.purl.post( + components=[{"purl": "pkg:npm/lodash@4.18.1"}], + org_slug="test-org", + strict=True, + ) + self.assertEqual(len(result), 1) + # Quota endpoints def test_quota_get_unit(self): """Test quota retrieval.""" diff --git a/uv.lock b/uv.lock index ae0b374..1db031d 100644 --- a/uv.lock +++ b/uv.lock @@ -1353,7 +1353,7 @@ wheels = [ [[package]] name = "socketdev" -version = "3.3.0" +version = "3.4.0" source = { editable = "." } dependencies = [ { name = "requests" },