diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7deb2..f92c405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 2.6.0 + +### Changed: scan comparison now polls the diff-scans endpoints + +- Diff mode no longer holds a single idle HTTP connection open while the API + computes the scan comparison. The CLI now creates a diff-scan resource + (`POST /orgs/{org}/diff-scans/from-ids`) and polls + `GET /orgs/{org}/diff-scans/{id}?cached=true` with short, bounded requests + until the comparison is ready (HTTP 200 instead of 202). This fixes + intermittent `Connection reset by peer` failures on the final comparison + step when scans take several minutes to compare and network middleboxes + (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout) + reap the idle connection (CE-354). +- Duplicate scan pairs are resolved after an HTTP 409 and then polled through + the same cached endpoint. This avoids automatically following the API's 302 + duplicate redirect with an uncached, potentially long-lived GET request. +- The change is transparent: no flags or workflow changes are needed. If the + org API token is missing the `diff-scans:create`, `diff-scans:list` or + `full-scans:list` scopes — or the new flow fails for any other reason — the + CLI logs a warning and falls back to the legacy streaming comparison. +- Requires `socketdev>=3.4.0`. + ## 2.5.9 ### Changed: bump pinned @coana-tech/cli to 15.10.3 diff --git a/pyproject.toml b/pyproject.toml index 25fbd17..a5ce9e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.5.9" +version = "2.6.0" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ @@ -16,7 +16,7 @@ dependencies = [ 'GitPython', 'packaging', 'python-dotenv', - "socketdev>=3.3.0,<4.0.0", + "socketdev>=3.4.0,<4.0.0", "bs4>=0.0.2", "markdown>=3.10", "brotli>=1.0.9; platform_python_implementation == 'CPython'", diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 0189af7..2a2ecb9 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.5.9' +__version__ = '2.6.0' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index a372de4..bfe8bcb 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -15,7 +15,7 @@ from socketsecurity.config import CliConfig from socketdev import socketdev from socketdev.exceptions import APIFailure -from socketdev.fullscans import FullScanParams, SocketArtifact +from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact from socketdev.org import Organization from socketdev.repos import RepositoryInfo import copy @@ -92,6 +92,25 @@ FULL_SCAN_UPLOAD_MAX_ATTEMPTS = len(FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS) FULL_SCAN_UPLOAD_BACKOFF_JITTER_SECONDS = 2.0 +# Diff-scan polling policy. The legacy scan comparison (fullscans.stream_diff) holds a +# single HTTP connection open, fully idle, while the backend computes the diff; network +# middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to +# 4 minutes) kill that connection with a RST, surfacing as an intermittent +# ConnectionResetError on large scans (CE-354). The diff-scans flow instead creates a +# diff-scan resource and polls its cached endpoint with short bounded requests: the API +# answers 202 while the comparison is still computing and 200 with the result once it is +# ready, so no connection is ever idle long enough to be reaped. +# +# Each poll consumes 1 unit of API quota, so the interval backs off toward +# DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take +# minutes to compute. The timeout is a backstop against a diff scan that never +# completes; on expiry (or any other failure of this flow) the caller falls back to the +# legacy streaming comparison rather than failing the scan outright. +DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 +DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0 +DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 +DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0 + def _humanize_alert_type(alert_type: str) -> str: """Convert a camelCase/PascalCase alert type into a Title-Cased label. @@ -1303,6 +1322,120 @@ def get_license_text_via_purl(self, packages: dict[str, Package], batch_size: in return packages + def get_diff_scan_artifacts( + self, + head_full_scan_id: str, + new_full_scan_id: str + ) -> DiffArtifacts: + """Compare two full scans via the diff-scans endpoints, polling for the result. + + Creates a diff-scan resource from the two full scan IDs, then polls + ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the + computed comparison (200) instead of a processing status (202). Unlike the + legacy ``fullscans.stream_diff`` call, no request is ever left idle while + the backend computes, so the comparison survives network idle timeouts + (CE-354). See the DIFF_SCAN_POLL_* constants for the polling policy. + + Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` + and ``full-scans:list`` scopes; callers are expected to catch failures and + fall back to the legacy streaming comparison. + + Note that cached diff-scan responses always embed per-package license + details (the API ignores ``omit_license_details`` when ``cached=true``), + so unlike the legacy streaming comparison there is no lean-response + option here; see the comment on ``poll_params`` below. + + Args: + head_full_scan_id: The before/base full scan ID + new_full_scan_id: The after/head full scan ID + + Returns: + DiffArtifacts with the added/removed/unchanged/replaced/updated lists + """ + create_params = { + "before": head_full_scan_id, + "after": new_full_scan_id, + "description": f"Socket Security CLI v{__version__} scan comparison", + } + try: + result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params) + diff_scan = result.get("diff_scan") or {} + response_summary = result + except APIFailure as error: + if error.status_code != 409: + raise + + # Do not use on_duplicate=redirect here. The SDK follows that 302 + # automatically with a GET that lacks cached=true, which can leave + # the connection idle while an existing diff scan is still computing. + # Resolve the duplicate resource explicitly so every result fetch + # continues through the bounded cached polling path below. + existing = self.sdk.diffscans.list( + self.config.org_slug, + params={ + "before_full_scan_id": head_full_scan_id, + "after_full_scan_id": new_full_scan_id, + "per_page": 1, + }, + ) + matches = existing.get("results") or [] + diff_scan = matches[0] if matches else {} + response_summary = existing + + diff_scan_id = diff_scan.get("id") + if not diff_scan_id: + raise Exception( + "Error creating or resolving diff scan: " + f"unexpected response: {str(response_summary)[:500]}" + ) + artifacts_dict = diff_scan.get("artifacts") + + # cached=true is the polling contract (202 while computing, 200 when + # ready). The API ignores omit_license_details when cached=true - cached + # results always embed license details - so there is no lean-response + # option on this path (unlike stream_diff with + # include_license_details=false, the CE-224 mitigation). If that extra + # payload ever gets a response truncated on a huge dependency tree, + # response.json() fails and the caller falls back to the legacy + # streaming comparison, which still requests the lean payload. + poll_params = {"cached": "true"} + deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS + interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS + while artifacts_dict is None: + try: + response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params) + except APIFailure as error: + if not error.is_transient_error(): + raise + # A dropped/timed-out poll is retryable: the diff scan keeps + # computing server-side regardless of what happens to any one poll. + log.warning( + f"Transient error polling diff scan {diff_scan_id} " + f"({type(error).__name__}), retrying in {interval:.0f}s" + ) + response = {"status": "processing"} + if response.get("status") != "processing": + scan = response.get("diff_scan") or {} + if scan.get("artifacts") is None: + raise Exception( + f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}" + ) + artifacts_dict = scan["artifacts"] + break + if time.monotonic() >= deadline: + raise Exception( + f"Timed out waiting for diff scan {diff_scan_id} after " + f"{DIFF_SCAN_POLL_TIMEOUT_SECONDS:.0f} seconds" + ) + log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s") + time.sleep(interval) + interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS) + + return DiffArtifacts.from_dict({ + key: artifacts_dict.get(key) or [] + for key in ("added", "removed", "unchanged", "replaced", "updated") + }) + def get_added_and_removed_packages( self, head_full_scan_id: str, @@ -1315,8 +1448,12 @@ def get_added_and_removed_packages( Args: head_full_scan_id: Previous scan (maybe None if first scan) new_full_scan_id: New scan just created - include_license_details: Whether to ask the diff endpoint to embed - per-package license attribution/details in the response. + include_license_details: Whether to ask the *legacy streaming* diff + endpoint to embed per-package license attribution/details in the + response. Only consulted on the fallback path: the primary + diff-scans path always receives embedded license details, since + the API ignores ``omit_license_details`` for cached reads (see + get_diff_scan_artifacts). Defaults to ``False`` on purpose. The diff endpoint exists to compare alerts between two scans; the license fields it can embed @@ -1343,39 +1480,55 @@ def get_added_and_removed_packages( log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}") diff_start = time.time() + diff_artifacts = None try: - diff_report = ( - self.sdk.fullscans.stream_diff( - self.config.org_slug, - head_full_scan_id, - new_full_scan_id, - use_types=True, - include_license_details=str(include_license_details).lower() - ).data + diff_artifacts = self.get_diff_scan_artifacts( + head_full_scan_id, + new_full_scan_id ) - except APIFailure as e: - log.error(f"API Error: {e}") - if self.cli_config and self.cli_config.disable_blocking: - sys.exit(0) - sys.exit(1) - except Exception as e: - import traceback - log.error(f"Error getting diff report: {str(e)}") - log.error(f"Stack trace:\n{traceback.format_exc()}") - raise + except Exception as error: + # SDK error messages can span many lines (path + response headers); the + # first line carries the status, which is all the warning needs. + error_summary = str(error).strip().splitlines()[0] if str(error).strip() else "" + log.warning( + f"Diff scan comparison failed with {type(error).__name__}({error_summary}), " + "falling back to the streaming scan comparison" + ) + + if diff_artifacts is None: + try: + diff_artifacts = ( + self.sdk.fullscans.stream_diff( + self.config.org_slug, + head_full_scan_id, + new_full_scan_id, + use_types=True, + include_license_details=str(include_license_details).lower() + ).data.artifacts + ) + except APIFailure as e: + log.error(f"API Error: {e}") + if self.cli_config and self.cli_config.disable_blocking: + sys.exit(0) + sys.exit(1) + except Exception as e: + import traceback + log.error(f"Error getting diff report: {str(e)}") + log.error(f"Stack trace:\n{traceback.format_exc()}") + raise diff_end = time.time() log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds") log.info("Diff report artifact counts:") - log.info(f"Added: {len(diff_report.artifacts.added)}") - log.info(f"Removed: {len(diff_report.artifacts.removed)}") - log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}") - log.info(f"Replaced: {len(diff_report.artifacts.replaced)}") - log.info(f"Updated: {len(diff_report.artifacts.updated)}") - - added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated - removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced - unchanged_artifacts = diff_report.artifacts.unchanged + log.info(f"Added: {len(diff_artifacts.added)}") + log.info(f"Removed: {len(diff_artifacts.removed)}") + log.info(f"Unchanged: {len(diff_artifacts.unchanged)}") + log.info(f"Replaced: {len(diff_artifacts.replaced)}") + log.info(f"Updated: {len(diff_artifacts.updated)}") + + added_artifacts = diff_artifacts.added + diff_artifacts.updated + removed_artifacts = diff_artifacts.removed + diff_artifacts.replaced + unchanged_artifacts = diff_artifacts.unchanged added_packages: Dict[str, Package] = {} removed_packages: Dict[str, Package] = {} diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 381c2c3..ae6b10c 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -87,6 +87,22 @@ def stream_diff_response(data_dir, load_json): }) +@pytest.fixture +def diff_scan_get_response(data_dir, load_json): + """GET /orgs/{org}/diff-scans/{id} response built from the stream_diff fixture. + + The diff-scans endpoint returns the same artifact shape as the legacy + streaming diff, wrapped in a diff_scan object. + """ + json_data = load_json(data_dir / "fullscans" / "diff" / "stream_diff.json") + return { + "diff_scan": { + "id": "diff-scan-123", + "artifacts": json_data["data"]["artifacts"], + } + } + + @@ -138,6 +154,7 @@ def mock_sdk_with_responses( new_scan_metadata, new_scan_stream, stream_diff_response, + diff_scan_get_response, create_full_scan_response, ): sdk = mock_socket_sdk.return_value @@ -173,4 +190,8 @@ def mock_sdk_with_responses( lambda org_slug, head_id, new_id, **kwargs: stream_diff_response ) + # Diff-scans endpoints (primary scan-comparison path) + sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-123"}} + sdk.diffscans.get.return_value = diff_scan_get_response + return sdk diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py new file mode 100644 index 0000000..7c98a51 --- /dev/null +++ b/tests/core/test_diff_scan_polling.py @@ -0,0 +1,122 @@ +"""Tests for the diff-scans polling scan comparison (CE-354). + +The comparison must never hold an idle connection open: it creates a diff-scan +resource and polls the cached endpoint (202 while processing, 200 when ready), +falling back to the legacy streaming diff if the new flow is unavailable. +""" +import pytest +from socketdev.exceptions import APIConnectionError, APIFailure + +import socketsecurity.core as core_module +from socketsecurity.core import Core +from socketsecurity.core.socket_config import SocketConfig + + +@pytest.fixture +def core(mock_sdk_with_responses): + config = SocketConfig(api_key="test_key") + return Core(config=config, sdk=mock_sdk_with_responses) + + +@pytest.fixture +def no_sleep(mocker): + return mocker.patch("socketsecurity.core.time.sleep") + + +def test_polls_until_diff_scan_ready(core, diff_scan_get_response, no_sleep): + """202 processing responses are polled through until the 200 result arrives.""" + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing, processing, diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 3 + assert no_sleep.call_count == 2 # slept between polls, never during them + assert len(artifacts.added) > 0 + + +def test_poll_interval_backs_off(core, diff_scan_get_response, no_sleep, monkeypatch): + """The poll interval grows toward the max so long comparisons stay quota-friendly.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS", 4.0) + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS", 10.0) + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing] * 4 + [diff_scan_get_response] + + core.get_diff_scan_artifacts("head", "new") + + waits = [call.args[0] for call in no_sleep.call_args_list] + assert waits == [4.0, 6.0, 9.0, 10.0] # 1.5x backoff, capped at the max + + +def test_transient_poll_error_is_retried(core, diff_scan_get_response, no_sleep): + """A dropped poll doesn't abandon the flow - the diff keeps computing server-side.""" + core.sdk.diffscans.get.side_effect = [APIConnectionError("reset"), diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 2 + assert len(artifacts.added) > 0 + + +def test_non_transient_poll_error_raises(core, no_sleep): + """Deterministic API errors (e.g. 403 missing scopes) propagate to the caller.""" + core.sdk.diffscans.get.side_effect = APIFailure("forbidden", status_code=403) + + with pytest.raises(APIFailure): + core.get_diff_scan_artifacts("head", "new") + + +def test_poll_timeout_raises(core, no_sleep, monkeypatch): + """A diff scan that never completes hits the polling backstop.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_TIMEOUT_SECONDS", 0.0) + core.sdk.diffscans.get.return_value = {"status": "processing", "id": "diff-scan-123"} + + with pytest.raises(Exception, match="Timed out waiting for diff scan"): + core.get_diff_scan_artifacts("head", "new") + + +def test_duplicate_conflict_uses_cached_polling(core, diff_scan_get_response): + """A duplicate is resolved explicitly so the SDK cannot follow an uncached redirect.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure( + "duplicate", status_code=409 + ) + core.sdk.diffscans.list.return_value = { + "results": [{"id": "existing-diff-scan"}], + } + + artifacts = core.get_diff_scan_artifacts("head", "new") + + create_params = core.sdk.diffscans.create_from_ids.call_args.args[1] + assert "on_duplicate" not in create_params + core.sdk.diffscans.list.assert_called_once_with( + core.config.org_slug, + params={ + "before_full_scan_id": "head", + "after_full_scan_id": "new", + "per_page": 1, + }, + ) + core.sdk.diffscans.get.assert_called_once_with( + core.config.org_slug, + "existing-diff-scan", + params={"cached": "true"}, + ) + assert len(artifacts.added) > 0 + + +def test_fallback_to_streaming_diff_on_failure(core): + """If the diff-scans flow fails (e.g. token missing the diff-scans scopes), + the comparison falls back to the legacy streaming diff transparently.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) + + added, removed, all_packages = core.get_added_and_removed_packages("head", "new") + + core.sdk.fullscans.stream_diff.assert_called_once_with( + core.config.org_slug, + "head", + "new", + use_types=True, + include_license_details="false", + ) + assert "dp3" in added + assert "dp2" in removed diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 9b1ce44..9edcbe9 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -225,19 +225,27 @@ def test_get_added_and_removed_packages(core): """Test getting added and removed packages between two scans""" # Get two different scans to compare added, removed, all_packages = core.get_added_and_removed_packages("head", "new") - - # Verify SDK was called correctly. - # include_license_details defaults to "false": the diff path never consumes - # embedded license data (license artifacts come from the PURL endpoint), so - # requesting it only bloats the response and risks the truncation - # crash on large repos. - core.sdk.fullscans.stream_diff.assert_called_once_with( + + # Verify SDK was called correctly: the comparison goes through the diff-scans + # endpoints (create + poll) rather than the legacy streaming diff, so no + # connection is left idle while the backend computes (CE-354). + create_args = core.sdk.diffscans.create_from_ids.call_args + assert create_args[0][0] == core.config.org_slug + create_params = create_args[0][1] + assert create_params["before"] == "head" + assert create_params["after"] == "new" + assert "on_duplicate" not in create_params + + # cached=true is the polling contract (202 while computing, 200 when ready). + # No omit_license_details param: the API ignores it for cached reads (cached + # results always embed license details), so sending it would only suggest a + # leanness guarantee this path doesn't have. + core.sdk.diffscans.get.assert_called_once_with( core.config.org_slug, - "head", - "new", - use_types=True, - include_license_details="false", + "diff-scan-123", + params={"cached": "true"}, ) + core.sdk.fullscans.stream_diff.assert_not_called() # Verify the results # Added packages @@ -252,7 +260,13 @@ def test_get_added_and_removed_packages(core): assert "pypi/direct_package_1@1.6.0" in all_packages # Unchanged package is in full package map def test_get_added_and_removed_packages_license_override(core): - """The include_license_details override seam still works when explicitly requested.""" + """include_license_details only governs the legacy fallback path now: the + diff-scans path always receives embedded license details (the API ignores + omit_license_details for cached reads), so the seam must survive through to + the stream_diff call when the primary path is unavailable.""" + from socketdev.exceptions import APIFailure + + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) core.get_added_and_removed_packages("head", "new", include_license_details=True) core.sdk.fullscans.stream_diff.assert_called_once_with(