fix(purl): expose fail-open batch params and harden dedupe

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketdev"
-version = "3.3.0"
+version = "3.4.0"
 requires-python = ">= 3.9"
 dependencies = [
     'requests',

socketdev/core/dedupe.py

@@ -11,8 +11,8 @@ def normalize_file_path(path: str) -> str:
     @staticmethod
     def alert_key(alert: dict) -> tuple:
         return (
-            alert["type"],
-            alert["severity"],
+            alert.get("type"),
+            alert.get("severity"),
             alert.get("category"),
             Dedupe.normalize_file_path(alert.get("file")),
             alert.get("start"),
@@ -23,8 +23,8 @@ def alert_key(alert: dict) -> tuple:
     def consolidate_and_merge_alerts(package_group: List[Dict[str, Any]]) -> Dict[str, Any]:
         def alert_identity(alert: dict) -> tuple:
             return (
-                alert["type"],
-                alert["severity"],
+                alert.get("type"),
+                alert.get("severity"),
                 alert.get("category"),
                 Dedupe.normalize_file_path(alert.get("file")),
                 alert.get("start"),
@@ -41,14 +41,17 @@ def alert_identity(alert: dict) -> tuple:
                 identity = alert_identity(alert)
 
                 if identity not in alert_map:
-                    # Build alert dict with only fields that exist in the original alert
+                    # Build alert dict with only fields that exist in the original alert.
+                    # Use .get() for key/type/severity/action so synthetic status rows
+                    # (e.g. pendingScan/notFound), which are built server-side from a
+                    # minimal {type, key} base, don't raise KeyError here.
                     consolidated_alert = {
-                        "key": alert["key"],  # keep the first key seen
-                        "type": alert["type"],
-                        "severity": alert["severity"],
+                        "key": alert.get("key"),  # keep the first key seen
+                        "type": alert.get("type"),
+                        "severity": alert.get("severity"),
                         "releases": [release],
                         "props": alert.get("props", []),
-                        "action": alert["action"]
+                        "action": alert.get("action")
                     }
 
                     # Only include optional fields if they exist in the original alert

socketdev/exceptions.py

@@ -78,3 +78,22 @@ class APIBadGateway(APIFailure):
 
     def __init__(self, *args):
         super().__init__(*args, status_code=502)
+
+
+class APIPartialResponse(APIFailure):
+    """Raised by ``purl.post(strict=True)`` when the batch response omits requested inputs.
+
+    The batch purl API is fail-open: input purls whose resolution/analysis has not
+    completed are silently dropped from the response unless the caller opts in via
+    ``alerts=True`` (synthetic ``pendingScan``/``notFound`` rows) or ``poll=True`` (a
+    bounded fail-closed wait). ``strict=True`` turns that silent omission into this
+    explicit error so callers get a first-class "partial batch" signal without having
+    to diff the response themselves.
+
+    The ``missing`` attribute holds the requested purls that were absent from the
+    response (the HTTP call itself succeeded, so there is no status code).
+    """
+
+    def __init__(self, *args, missing=None):
+        super().__init__(*args)
+        self.missing = list(missing or [])

socketdev/purl/__init__.py

@@ -1,7 +1,9 @@
 import json
 import urllib.parse
 import warnings
+from typing import Optional
 from socketdev.log import log
+from socketdev.exceptions import APIPartialResponse
 from ..core.dedupe import Dedupe
 
 
@@ -14,8 +16,55 @@ def post(
         license: str = "false",
         components: list = None,
         org_slug: str = None,
+        poll: Optional[bool] = None,
+        timeout_sec: Optional[int] = None,
+        alerts: Optional[bool] = None,
+        purl_errors: Optional[bool] = None,
+        strict: bool = False,
         **kwargs,
     ) -> list:
+        """POST a batch of purls to the Socket batch purl endpoint and return deduped rows.
+
+        The batch purl API (``POST /v0/purl`` and ``POST /v0/orgs/{slug}/purl``) defaults
+        to **fail-open**: any input purl whose resolution/analysis has not finished is
+        **silently omitted** from the response. A naive caller therefore cannot tell
+        "this version is clean" apart from "this version was dropped from the response".
+        The parameters below opt into the server behaviors that make omissions visible.
+
+        Args:
+            license: ``"true"``/``"false"`` — request license information (stringly-typed
+                to match the query param the API expects).
+            components: list of component dicts to score, e.g. ``[{"purl": "pkg:npm/lodash@4.18.1"}]``.
+            org_slug: organization slug. When provided, routes to the org-scoped endpoint
+                ``POST /v0/orgs/{org_slug}/purl``; otherwise the deprecated ``POST /v0/purl``.
+            poll: opt into a fail-closed bounded wait for pending analysis (``poll=True`` →
+                ``poll=true`` query param). ``None`` omits the param (server default).
+            timeout_sec: bound in seconds for the ``poll`` wait (``→ timeoutSec``). The
+                server may cap this via a feature flag. ``None`` omits the param.
+            alerts: when ``True`` (``→ alerts=true``), the server emits synthetic
+                ``pendingScan``/``notFound`` status rows instead of silently omitting
+                unresolved inputs, so callers can distinguish "no data yet" from "clean".
+            purl_errors: when ``True`` (``→ purlErrors``), the server includes per-purl
+                error rows for malformed/unresolvable inputs. ``None`` omits the param.
+            strict: client-side guard. When ``True``, compares the ``purl`` of each
+                requested component against the ``inputPurl``/``purl`` of the returned
+                rows and raises :class:`~socketdev.exceptions.APIPartialResponse` (with a
+                ``missing`` list) if any requested purl is absent from the response. This
+                surfaces partial batches even without ``alerts=True``. Only components that
+                carry a ``purl`` string are checked.
+            **kwargs: forwarded verbatim into the query string (back-compat passthrough for
+                any params not yet promoted to first-class arguments).
+
+        Returns:
+            A deduped list of result rows. When ``alerts=True``, unresolved inputs appear
+            as synthetic rows carrying ``pendingScan``/``notFound`` alerts rather than being
+            omitted. On a non-200 response, logs the error and returns ``[]`` (callers that
+            need to fail closed should treat ``[]`` as an error).
+
+        Raises:
+            APIPartialResponse: if ``strict=True`` and one or more requested component purls
+                are missing from the response.
+        """
         if org_slug is None:
             warnings.warn(
                 "Calling purl.post() without org_slug uses the deprecated POST /v0/purl endpoint. "
@@ -31,25 +80,83 @@ def post(
         query_args = {
             "license": license,
         }
+        # Promote the typed params into query args only when explicitly set, so existing
+        # callers keep the server's fail-open default (None => omit the param entirely).
+        if poll is not None:
+            query_args["poll"] = "true" if poll else "false"
+        if timeout_sec is not None:
+            query_args["timeoutSec"] = str(timeout_sec)
+        if alerts is not None:
+            query_args["alerts"] = "true" if alerts else "false"
+        if purl_errors is not None:
+            query_args["purlErrors"] = "true" if purl_errors else "false"
         if kwargs:
             query_args.update(kwargs)
         params = urllib.parse.urlencode(query_args)
         path += params
         response = self.api.do_request(path=path, payload=purls, method="POST")
         if response.status_code == 200:
-            purl = []
+            artifact_rows = []
+            stream_records = []
             result = response.text
             result = result.strip('"').strip()
             for line in result.split("\n"):
                 if line and line != '"':
                     try:
                         item = json.loads(line)
-                        purl.append(item)
+                        if isinstance(item, dict) and item.get("_type") in {
+                            "purlError",
+                            "summary",
+                        }:
+                            stream_records.append(item)
+                        else:
+                            artifact_rows.append(item)
                     except json.JSONDecodeError:
                         continue
-            purl_deduped = Dedupe.dedupe(purl, batched=True)
+            purl_deduped = Dedupe.dedupe(artifact_rows, batched=True)
+            purl_deduped.extend(stream_records)
+            if strict:
+                self._raise_on_missing(components, purl_deduped)
             return purl_deduped
 
         log.error(f"Error posting {components} to the Purl API: {response.status_code}")
         log.error(response.text)
         return []
+
+    @staticmethod
+    def _raise_on_missing(components: list, results: list) -> None:
+        """Raise APIPartialResponse if any requested component purl is absent from results.
+
+        Only components exposing a ``purl`` string are checked; the batch API echoes the
+        request identifier back as ``inputPurl`` (falling back to ``purl``), including
+        under ``value`` for typed ``purlError`` stream records.
+        """
+        requested = [
+            c["purl"]
+            for c in components
+            if isinstance(c, dict) and isinstance(c.get("purl"), str)
+        ]
+        if not requested:
+            return
+        returned = set()
+        for row in results:
+            if not isinstance(row, dict):
+                continue
+            for field in ("inputPurl", "purl"):
+                value = row.get(field)
+                if isinstance(value, str):
+                    returned.add(value)
+            record_value = row.get("value")
+            if isinstance(record_value, dict):
+                input_purl = record_value.get("inputPurl")
+                if isinstance(input_purl, str):
+                    returned.add(input_purl)
+        missing = [purl for purl in requested if purl not in returned]
+        if missing:
+            raise APIPartialResponse(
+                "purl.post(strict=True): the batch response omitted "
+                f"{len(missing)} of {len(requested)} requested purls "
+                "(fail-open: unresolved inputs are dropped unless alerts=True/poll=True): "
+                f"{missing}",
+                missing=missing,
+            )

socketdev/version.py

@@ -1 +1 @@
-__version__ = "3.3.0"
+__version__ = "3.4.0"

tests/unit/test_all_endpoints_unit.py

@@ -401,6 +401,133 @@ def test_purl_post_unit_legacy_path(self):
         self.assertIn("/purl", call_args[0][1])
         self.assertNotIn("/orgs/", call_args[0][1])
 
+    def _mock_purl_ndjson(self, ndjson):
+        """Mock a 200 NDJSON purl response and return the mock."""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.headers = {'content-type': 'application/x-ndjson'}
+        mock_response.text = ndjson
+        self.mock_requests.request.return_value = mock_response
+        return mock_response
+
+    def test_purl_post_first_class_params_query_string(self):
+        """poll/timeout_sec/alerts/purl_errors map to the expected query params."""
+        self._mock_purl_ndjson(
+            '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", '
+            '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}'
+        )
+
+        self.sdk.purl.post(
+            license="false",
+            components=[{"purl": "pkg:npm/lodash@4.18.1"}],
+            org_slug="test-org",
+            poll=True,
+            timeout_sec=120,
+            alerts=True,
+            purl_errors=False,
+        )
+
+        url = self.mock_requests.request.call_args[0][1]
+        self.assertIn("poll=true", url)
+        self.assertIn("timeoutSec=120", url)
+        self.assertIn("alerts=true", url)
+        self.assertIn("purlErrors=false", url)
+
+    def test_purl_post_omits_unset_params(self):
+        """None-valued typed params are omitted so the API's fail-open default is preserved."""
+        self._mock_purl_ndjson(
+            '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", '
+            '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}'
+        )
+
+        self.sdk.purl.post(components=[{"purl": "pkg:npm/lodash@4.18.1"}], org_slug="test-org")
+
+        url = self.mock_requests.request.call_args[0][1]
+        self.assertNotIn("poll=", url)
+        self.assertNotIn("timeoutSec=", url)
+        self.assertNotIn("alerts=", url)
+        self.assertNotIn("purlErrors=", url)
+
+    def test_purl_post_synthetic_pending_scan_row(self):
+        """A synthetic pendingScan row (no severity/action) parses without raising KeyError."""
+        # Synthetic status alerts are built server-side from a minimal {type, key} base.
+        self._mock_purl_ndjson(
+            '{"inputPurl": "pkg:npm/newpkg@0.0.1", "purl": "pkg:npm/newpkg@0.0.1", '
+            '"type": "npm", "name": "newpkg", "version": "0.0.1", '
+            '"alerts": [{"type": "pendingScan", "key": "abc123"}]}'
+        )
+
+        result = self.sdk.purl.post(
+            components=[{"purl": "pkg:npm/newpkg@0.0.1"}],
+            org_slug="test-org",
+            alerts=True,
+        )
+
+        self.assertEqual(len(result), 1)
+        alert = result[0]["alerts"][0]
+        self.assertEqual(alert["type"], "pendingScan")
+        self.assertEqual(alert["key"], "abc123")
+        # Missing fields are surfaced as None rather than raising.
+        self.assertIsNone(alert["severity"])
+        self.assertIsNone(alert["action"])
+
+    def test_purl_post_preserves_purl_error_record(self):
+        """purlError stream records bypass artifact deduplication."""
+        error_row = {
+            "_type": "purlError",
+            "value": {
+                "error": "package_not_found",
+                "inputPurl": "pkg:npm/missing@1.0.0",
+            },
+        }
+        self._mock_purl_ndjson(json.dumps(error_row))
+
+        result = self.sdk.purl.post(
+            components=[{"purl": "pkg:npm/missing@1.0.0"}],
+            org_slug="test-org",
+            purl_errors=True,
+            strict=True,
+        )
+
+        self.assertEqual(result, [error_row])
+        url = self.mock_requests.request.call_args[0][1]
+        self.assertIn("purlErrors=true", url)
+
+    def test_purl_post_strict_raises_on_missing(self):
+        """strict=True raises APIPartialResponse listing purls dropped from the response."""
+        from socketdev.exceptions import APIPartialResponse
+
+        # Requested two purls; the fail-open API only returned one.
+        self._mock_purl_ndjson(
+            '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", '
+            '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}'
+        )
+
+        with self.assertRaises(APIPartialResponse) as ctx:
+            self.sdk.purl.post(
+                components=[
+                    {"purl": "pkg:npm/lodash@4.18.1"},
+                    {"purl": "pkg:npm/dropped@0.0.1"},
+                ],
+                org_slug="test-org",
+                strict=True,
+            )
+        self.assertEqual(ctx.exception.missing, ["pkg:npm/dropped@0.0.1"])
+
+    def test_purl_post_strict_passes_when_complete(self):
+        """strict=True returns normally when every requested purl is present."""
+        self._mock_purl_ndjson(
+            '{"inputPurl": "pkg:npm/lodash@4.18.1", "purl": "pkg:npm/lodash@4.18.1", '
+            '"type": "npm", "name": "lodash", "version": "4.18.1", "valid": true, "alerts": []}'
+        )
+
+        result = self.sdk.purl.post(
+            components=[{"purl": "pkg:npm/lodash@4.18.1"}],
+            org_slug="test-org",
+            strict=True,
+        )
+        self.assertEqual(len(result), 1)
+
     # Quota endpoints
     def test_quota_get_unit(self):
         """Test quota retrieval."""