Raise failure on SBOM fetch errors #493
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| e2e: | |
| # Skip e2e on: | |
| # - PRs from forks (no secrets) | |
| # - Dependabot PRs (no secrets, and dependency-bump risk is already | |
| # covered by dependency-review.yml's Socket Firewall smoke jobs) | |
| if: >- | |
| (github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.full_name == github.repository) && | |
| github.event.pull_request.user.login != 'dependabot[bot]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: scan | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-npm | |
| --disable-blocking | |
| --enable-debug | |
| validate: tests/e2e/validate-scan.sh | |
| - name: sarif | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-npm | |
| --sarif-file /tmp/results.sarif | |
| --disable-blocking | |
| validate: tests/e2e/validate-sarif.sh | |
| - name: reachability | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-npm | |
| --reach | |
| --disable-blocking | |
| --enable-debug | |
| validate: tests/e2e/validate-reachability.sh | |
| setup-node: "true" | |
| # The tier-1 reachability backend intermittently returns empty | |
| # results while the CLI reports success (ENG-5093). The probe | |
| # exits 0 when the facts file has alerted components; anything | |
| # else is retried before validation fails the job. | |
| retry-probe: bash tests/e2e/reach-facts-probe.sh tests/e2e/fixtures/simple-npm | |
| - name: gitlab | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-npm | |
| --enable-gitlab-security | |
| --disable-blocking | |
| validate: tests/e2e/validate-gitlab.sh | |
| - name: json | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-npm | |
| --enable-json | |
| --disable-blocking | |
| validate: tests/e2e/validate-json.sh | |
| - name: pypi | |
| args: >- | |
| --target-path tests/e2e/fixtures/simple-pypi | |
| --disable-blocking | |
| --enable-debug | |
| validate: tests/e2e/validate-scan.sh | |
| name: e2e-${{ matrix.name }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: '3.12' | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| if: matrix.setup-node == 'true' | |
| with: | |
| node-version: '20' | |
| - name: Install CLI from local repo | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install . | |
| - name: Install uv | |
| if: matrix.setup-node == 'true' | |
| run: pip install uv | |
| - name: Run Socket CLI | |
| env: | |
| SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} | |
| RETRY_PROBE: ${{ matrix.retry-probe }} | |
| run: | | |
| set -o pipefail | |
| # Entries with retry-probe get up to 3 attempts: the probe exits 0 | |
| # when the scan output looks complete, and a run that fails it is | |
| # re-run on the assumption of a transient backend failure. A | |
| # persistent failure still reaches the validate step, which fails | |
| # the job with full context. Retries are surfaced as warning | |
| # annotations so flake frequency stays visible instead of being | |
| # silently absorbed. | |
| max_attempts=3 | |
| attempt=1 | |
| while :; do | |
| socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log | |
| [ -z "$RETRY_PROBE" ] && break | |
| if bash -c "$RETRY_PROBE"; then | |
| break | |
| fi | |
| if [ "$attempt" -ge "$max_attempts" ]; then | |
| echo "::warning title=e2e-${{ matrix.name }} incomplete results::output still fails the completeness probe after ${max_attempts} attempts; letting validation fail the job" | |
| break | |
| fi | |
| echo "::warning title=e2e-${{ matrix.name }} transient retry::attempt ${attempt} failed the completeness probe (suspected backend transient, see ENG-5093); retrying" | |
| echo "e2e-${{ matrix.name }}: retry after attempt ${attempt} — completeness probe failed (suspected transient)" >> "$GITHUB_STEP_SUMMARY" | |
| attempt=$((attempt+1)) | |
| sleep 30 | |
| done | |
| - name: Validate results | |
| env: | |
| SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} | |
| run: bash ${{ matrix.validate }} | |
| - name: Upload diagnostics on failure | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-${{ matrix.name }}-diagnostics-attempt${{ github.run_attempt }} | |
| path: | | |
| /tmp/e2e-output.log | |
| /tmp/*.sarif | |
| tests/e2e/fixtures/simple-npm/.socket.facts.json | |
| tests/e2e/fixtures/simple-pypi/.socket.facts.json | |
| gl-*.json | |
| license_output.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| # Branch protection requires the e2e-* checks, but the `e2e` job above is | |
| # skipped on PRs that can't access repository secrets -- fork PRs and | |
| # Dependabot PRs. A job skipped via a job-level `if` never expands its | |
| # matrix, so the e2e-* check contexts are never created and the required | |
| # checks sit at "Expected -- Waiting for status to be reported" forever, | |
| # permanently blocking merge. | |
| # | |
| # This bypass reports a green status under the SAME e2e-* check names for | |
| # exactly those PRs, satisfying branch protection without running the real | |
| # tests (which need SOCKET_CLI_API_TOKEN). Its `if` is the precise negation | |
| # of the e2e job's run condition, so the two are mutually exclusive: any | |
| # given PR runs one or the other, never both, and never neither. | |
| # | |
| # Dependency-bump risk on these PRs is still covered by dependency-review.yml's | |
| # Socket Firewall smoke jobs, which run without repository secrets. | |
| e2e-bypass: | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| (github.event.pull_request.head.repo.full_name != github.repository || | |
| github.event.pull_request.user.login == 'dependabot[bot]') | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| name: [scan, sarif, reachability, gitlab, json, pypi] | |
| name: e2e-${{ matrix.name }} | |
| steps: | |
| - name: Report skip status | |
| run: | | |
| echo "Skipping e2e-${{ matrix.name }} for a PR without repository secrets" | |
| echo "(fork or Dependabot). Dependency risk is covered by dependency-review.yml." |