castacks · pvkumara · Aug 21, 2026 · Aug 21, 2026 · Aug 21, 2026
diff --git a/.agents/skills/run-system-tests/SKILL.md b/.agents/skills/run-system-tests/SKILL.md
@@ -14,7 +14,7 @@ metadata:
 Use this skill when you need to:
 
 - Invoke the pytest system tests locally (via `airstack test`) or on CI (via `/pytest` PR comment or `workflow_dispatch`)
-- Diagnose a failing system test — interpret `summary.txt`, `results.xml`, `run_meta.json`, and `metrics.json` from `tests/results/<timestamp>/`
+- Diagnose a failing system test — interpret `summary.txt`, `results.xml`, schema-v2 `run_meta.json`, `metrics.json`, and bounded `diagnostics/` from `tests/results/<timestamp>/`
 - Compare metrics against a baseline run (`parse_metrics.py --baseline`) to confirm a regression or improvement
 - Add a new system test to `tests/`: pick the right mark, wire up `airstack_env` parametrization, and record metrics with `MetricsRecorder`
 
@@ -210,7 +210,7 @@ The workflow:
 2. Opens an in-progress GitHub Check Run on the PR's head SHA so the run shows up in the **Checks** tab (issue_comment events otherwise associate runs with the default branch)
 3. Runs pytest on a freshly-spawned ephemeral OSMO GPU pod (`runs-on: [self-hosted, airstack-ephemeral]`)
 4. Uploads `tests/results/` as artifact `test-results-<sha>-<run_id>` (90-day retention)
-5. The downstream `report` job runs `parse_metrics.py`, compares only a matching complete simulation baseline, posts the result, and finalizes the PR-head Check Run
+5. The downstream `report` job selects the newest matching complete simulation baseline, runs `parse_metrics.py`, posts the advisory comparison, and finalizes the PR-head Check Run
 6. Closes the Check Run with the final conclusion
 
 ### Why fork PRs are blocked
@@ -262,7 +262,7 @@ Keys follow `test_node_id → metric_key → {value, unit, direction, ...}`. Tim
 # Single-run report — markdown table, exits 0 always
 python tests/parse_metrics.py --current tests/results/2025-04-21_14-30-00/
 
-# Diff mode — side-by-side, exits 1 on regression
+# Comparison mode — side-by-side; numeric deltas are advisory
 python tests/parse_metrics.py \
   --current  tests/results/2025-04-21_14-30-00/ \
   --baseline tests/results/2025-04-20_09-00-00/ \
@@ -276,7 +276,11 @@ The report has three sections per test module:
 - **Sim publishing rates** — pivoted Hz aggregates per topic (`mean`, `start_mean`, `end_mean`, `min`, `max`) from the `sensors` mark (sim + robot streams)
 - **Compute usage** — pivoted CPU/mem/GPU per container
 
-Regressions exceeding `--threshold` (default 20%) are flagged `:red_circle:`; improvements beyond threshold get `:green_circle:`. CI fails only when both artifacts are complete and have the same simulation campaign fingerprint.
+Changes exceeding `--threshold` (default 20%) are flagged `:red_circle:` or
+`:green_circle:` for review. Numeric deltas never fail CI. Pytest assertions,
+infrastructure/prerequisite failures, missing artifacts, and report-parser
+errors remain blocking. The fingerprint includes normalized tests and all
+behavior-changing campaign options.
 
 When local-debugging a CI regression, download both artifacts (`test-results-<sha>-<run_id>` from the PR run and from the base branch's most recent run), unzip them under `tests/results/`, and run `parse_metrics.py` locally to see the same table the bot posted.
 
@@ -427,9 +431,9 @@ python tests/parse_metrics.py \
 ### Files to know
 
 - `tests/conftest.py` — pytest hooks + the `airstack_env` / `robot_autonomy_stack` fixtures (re-exports the harness API)
-- `tests/harness/` — helpers split by concern: `session`, `discovery`, `commands`, `containers`, `metrics` (`MetricsRecorder`), `run_meta`, `test_ids`, `sim`, `collection` (ordering)
+- `tests/harness/` — helpers split by concern: `session`, `discovery`, `commands`, `containers`, `metrics` (`MetricsRecorder`), `run_meta`, `baseline`, `diagnostics`, `test_ids`, `sim`, `collection` (ordering)
 - `tests/pytest.ini` — mark registration, log format
-- `tests/parse_metrics.py` — markdown reporter, regression diff
+- `tests/parse_metrics.py` — markdown reporter and advisory comparison
 - `tests/README.md` — user-facing docs (CLI options, output layout, CI/CD orchestrator)
 - `.github/workflows/system-tests.yml` — CI workflow with `/pytest` comment trigger
 - `.github/orchestrator/README.md` — ephemeral OSMO runner setup and worker-debug procedure

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -30,6 +30,14 @@ on:
         description: "Seconds for test_stable polling window"
         default: "120"
         required: false
+      trajectory_types:
+        description: "Fixed trajectories, comma-separated (e.g. Circle or Circle,Figure8)"
+        default: "Circle,Figure8,Racetrack,Line"
+        required: false
+      takeoff_velocities:
+        description: "Takeoff velocities, comma-separated (e.g. 0.5 or 0.5,1)"
+        default: "0.5"
+        required: false
       baseline_run_id:
         description: "Run ID to use as baseline for metric comparison (blank = latest successful run on main)"
         default: ""
@@ -58,8 +66,8 @@ jobs:
        startsWith(github.event.comment.body, '/pytest') &&
        contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association))
     concurrency:
-      group: system-tests-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
-      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+      group: system-tests-${{ github.event.pull_request.number || github.event.issue.number || github.run_id }}
+      cancel-in-progress: true
     timeout-minutes: 120
     # Adding any `permissions:` entry disables GITHUB_TOKEN's defaults, so
     # every scope used here has to be re-granted explicitly:
@@ -111,7 +119,6 @@ jobs:
             core.setOutput('base_ref', pr.data.base.ref);
 
       - name: Resolve tested revision identity
-        if: always()
         id: identity
         env:
           EVENT_NAME: ${{ github.event_name }}
@@ -121,7 +128,11 @@ jobs:
           EVENT_PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
           if [[ "$EVENT_NAME" == "issue_comment" ]]; then
-            echo "tested_sha=${COMMENT_HEAD_SHA:-$EVENT_SHA}" >> "$GITHUB_OUTPUT"
+            if [[ -z "$COMMENT_HEAD_SHA" ]]; then
+              echo "::error::Refusing /pytest run: PR head SHA was not resolved."
+              exit 1
+            fi
+            echo "tested_sha=$COMMENT_HEAD_SHA" >> "$GITHUB_OUTPUT"
             echo "pr_number=$COMMENT_PR_NUMBER" >> "$GITHUB_OUTPUT"
           else
             echo "tested_sha=$EVENT_SHA" >> "$GITHUB_OUTPUT"
@@ -141,6 +152,8 @@ jobs:
           INPUT_NUM_ROBOTS: ${{ inputs.num_robots }}
           INPUT_ITERATIONS: ${{ inputs.stress_iterations }}
           INPUT_STABLE: ${{ inputs.stable_duration }}
+          INPUT_TRAJECTORIES: ${{ inputs.trajectory_types }}
+          INPUT_TAKEOFF_VELOCITIES: ${{ inputs.takeoff_velocities }}
         run: |
           python3 <<'PYEOF'
           import os, shlex, sys
@@ -158,6 +171,10 @@ jobs:
                   args.extend(['--stress-iterations', it])
               if (st := os.environ.get('INPUT_STABLE', '').strip()):
                   args.extend(['--stable-duration', st])
+              if (trajectories := os.environ.get('INPUT_TRAJECTORIES', '').strip()):
+                  args.extend(['--trajectory-types', trajectories])
+              if (velocities := os.environ.get('INPUT_TAKEOFF_VELOCITIES', '').strip()):
+                  args.extend(['--takeoff-velocities', velocities])
           elif event == 'pull_request':
               # Automatic PR validation is deliberately build-scoped. Fast
               # Python unit tests run in unit-tests.yml; GPU simulation remains
@@ -369,17 +386,38 @@ jobs:
           SIM_INPUT: ${{ steps.parse.outputs.sim }}
           NO_IMAGE_BUILD: ${{ steps.parse.outputs.no_image_build }}
         run: |
+          mkdir -p tests/results
+          image_result=tests/results/image-preparation.json
+          image_outcome=already-present
+          pulled=()
+          retagged=()
+          built=()
+          missing=()
           profiles=desktop
           [[ ",$SIM_INPUT," == *,msairsim,* ]] && profiles="$profiles,ms-airsim"
           [[ ",$SIM_INPUT," == *,isaacsim,* ]] && profiles="$profiles,isaac-sim"
           export COMPOSE_PROFILES="$profiles"
           echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES (no_image_build=$NO_IMAGE_BUILD)"
+          declare -A present_before=()
+          while IFS= read -r img; do
+            [[ -z "$img" ]] && continue
+            if docker image inspect "$img" --format '{{.Id}}' >/dev/null 2>&1; then
+              present_before["$img"]=1
+            fi
+          done < <(docker compose -f docker-compose.yaml config --images)
 
           # Pull from registry; tolerate per-image failures so we can detect
           # what's still missing afterwards instead of aborting on the first
           # gap. `--progress=quiet` suppresses per-layer progress; errors
           # still surface on stderr.
           ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true
+          while IFS= read -r img; do
+            [[ -z "$img" || -n "${present_before[$img]:-}" ]] && continue
+            if docker image inspect "$img" --format '{{.Id}}' >/dev/null 2>&1; then
+              pulled+=("$img")
+              image_outcome=pulled-versioned
+            fi
+          done < <(docker compose -f docker-compose.yaml config --images)
 
           # VERSION tags miss on every PR. Seed from floating cache_* tags.
           cache_tag="$(grep -E '^CACHE_TAG=' .env 2>/dev/null | cut -d= -f2 | tr -d '"' || true)"
@@ -395,6 +433,8 @@ jobs:
             if docker pull --quiet "$cache_img"; then
               docker tag "$cache_img" "$img"
               echo "Retagged $cache_img -> $img"
+              retagged+=("$img")
+              image_outcome=cache-retagged
             else
               echo "Cache tag pull failed for $cache_img"
             fi
@@ -412,18 +452,38 @@ jobs:
             echo "Images still missing after pull/retag:"
             printf '  - %s\n' "${missing[@]}"
             if [[ "$NO_IMAGE_BUILD" == "true" ]]; then
+              IMAGE_OUTCOME=missing IMAGE_RESULT="$image_result" \
+                MISSING_IMAGES="$(printf '%s\n' "${missing[@]}")" \
+                PYTHONPATH=tests python3 -m harness.image_prep "$image_result"
               echo "::error::Pull-only mode (--no-image-build or -m build_packages) will not image-build. Run /pytest -m build_docker once, or omit --no-image-build."
               exit 1
             fi
             echo "Falling back to image-build"
             ./airstack.sh --progress=quiet image-build
+            built=("${missing[@]}")
+            image_outcome=locally-built
           else
             echo "All required images present after pull/retag — skipping build."
           fi
+          IMAGE_OUTCOME="$image_outcome" IMAGE_RESULT="$image_result" \
+            PULLED_IMAGES="$(printf '%s\n' "${pulled[@]}")" \
+            RETAGGED_IMAGES="$(printf '%s\n' "${retagged[@]}")" \
+            BUILT_IMAGES="$(printf '%s\n' "${built[@]}")" \
+            PYTHONPATH=tests python3 -m harness.image_prep "$image_result"
+          echo "### Image preparation: $image_outcome" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Record test-owned image preparation
+        if: ${{ steps.parse.outputs.skip_image_prep == 'true' }}
+        run: |
+          IMAGE_OUTCOME=delegated-to-build-docker PYTHONPATH=tests \
+            python3 -m harness.image_prep tests/results/image-preparation.json
+          echo "### Image preparation: delegated to build_docker tests" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Run tests
         env:
           AIRSTACK_ROOT: ${{ github.workspace }}
+          AIRSTACK_TESTED_SHA: ${{ steps.identity.outputs.tested_sha }}
+          AIRSTACK_PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
           DISPLAY: ""
           PYTEST_ARGS: ${{ steps.parse.outputs.pytest_args }}
         run: |
@@ -486,6 +546,7 @@ jobs:
     if: >
       always() &&
       needs.run-tests.result != 'skipped' &&
+      needs.run-tests.outputs.tested_sha != '' &&
       (needs.run-tests.result != 'cancelled' || github.event_name != 'pull_request')
     permissions:
       actions: read
@@ -505,7 +566,7 @@ jobs:
           python-version: "3.12"
 
       - name: Install report dependencies
-        run: pip install tabulate
+        run: pip install -r tests/report-requirements.txt
 
       - name: Resolve PR base branch
         if: github.event_name == 'issue_comment' || github.event_name == 'pull_request'
@@ -531,15 +592,19 @@ jobs:
       # the PR's base branch (e.g. develop or main).
       - name: Download baseline results (PR)
         if: github.event_name == 'issue_comment' || github.event_name == 'pull_request'
-        uses: dawidd6/action-download-artifact@v6
-        continue-on-error: true
-        with:
-          workflow: system-tests.yml
-          branch: ${{ steps.pr_ctx.outputs.base_ref }}
-          name_is_regexp: true
-          name: "test-results-.*"
-          path: baseline-results/
-          if_no_artifact_found: warn
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ steps.pr_ctx.outputs.base_ref }}
+        run: |
+          mkdir -p baseline-results
+          gh api --method GET \
+            "repos/${{ github.repository }}/actions/workflows/system-tests.yml/runs" \
+            -f branch="$BASE_REF" -f status=success -f per_page=20 \
+            --jq '.workflow_runs[].id' |
+          while read -r run_id; do
+            gh run download "$run_id" --repo "${{ github.repository }}" \
+              --pattern "test-results-*" --dir "baseline-results/$run_id" || true
+          done
 
       # Manual dispatch with explicit baseline run ID
       - name: Download baseline results (manual, explicit run ID)
@@ -560,29 +625,38 @@ jobs:
         if: >
           github.event_name == 'workflow_dispatch' &&
           inputs.baseline_run_id == ''
-        uses: dawidd6/action-download-artifact@v6
-        continue-on-error: true
-        with:
-          workflow: system-tests.yml
-          branch: main
-          name_is_regexp: true
-          name: "test-results-.*"
-          path: baseline-results/
-          if_no_artifact_found: warn
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p baseline-results
+          gh api --method GET \
+            "repos/${{ github.repository }}/actions/workflows/system-tests.yml/runs" \
+            -f branch=main -f status=success -f per_page=20 \
+            --jq '.workflow_runs[].id' |
+          while read -r run_id; do
+            gh run download "$run_id" --repo "${{ github.repository }}" \
+              --pattern "test-results-*" --dir "baseline-results/$run_id" || true
+          done
 
       - name: Locate result directories
         id: dirs
-        # Find the dir holding results.xml. Nesting depth differs by downloader:
-        # actions/download-artifact@v4 (single name) extracts straight into the
-        # path, while dawidd6/action-download-artifact@v6 with name_is_regexp
-        # wraps each artifact in a subdir named after it. `find` handles both.
+        # Find the current dir, then choose a baseline from the recent-run
+        # candidate tree by completed campaign fingerprint.
         run: |
           CURRENT_XML=$(find current-results/ -name results.xml 2>/dev/null | sort -r | head -1)
           [ -n "$CURRENT_XML" ] && echo "current=$(dirname "$CURRENT_XML")" >> "$GITHUB_OUTPUT"
 
-          BASELINE_XML=$(find baseline-results/ -name results.xml 2>/dev/null | sort -r | head -1)
-          if [ -n "$BASELINE_XML" ]; then
-            echo "baseline=$(dirname "$BASELINE_XML")" >> "$GITHUB_OUTPUT"
+          if [ -n "$CURRENT_XML" ] && [ -d baseline-results ]; then
+            CURRENT_DIR="$(dirname "$CURRENT_XML")"
+            BASELINE=$(PYTHONPATH=tests python3 - "$CURRENT_DIR" <<'PYEOF'
+          import sys
+          from pathlib import Path
+          from harness.baseline import select_baseline_path
+          selected = select_baseline_path(Path(sys.argv[1]), Path("baseline-results"))
+          print(selected or "")
+          PYEOF
+          )
+            echo "baseline=$BASELINE" >> "$GITHUB_OUTPUT"
           else
             echo "baseline=" >> "$GITHUB_OUTPUT"
           fi
@@ -602,8 +676,8 @@ jobs:
 
           Pass-rate and regression tables are suppressed because no completed test campaign is available.
           EOF
-            echo "parser_exit=0" >> "$GITHUB_OUTPUT"
-            exit 0
+            echo "parser_exit=2" >> "$GITHUB_OUTPUT"
+            exit 2
           fi
 
           set +e
@@ -653,14 +727,10 @@ jobs:
             echo "_No metrics report generated._" >> "$GITHUB_STEP_SUMMARY"
           fi
 
-      - name: Fail on regression
+      - name: Fail on report integrity error
         if: steps.report.outcome == 'failure'
         run: |
-          if [ "${{ steps.report.outputs.parser_exit }}" = "1" ]; then
-            echo "::error::Metric regression detected — see the report above for details."
-          else
-            echo "::error::Metrics report generation failed — see the report step log."
-          fi
+          echo "::error::Metrics report generation failed — see the report step log."
           exit 1
 
       - name: Finalize check on PR head

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -35,6 +35,8 @@ jobs:
       - name: Run unit tests
         env:
           AIRSTACK_ROOT: ${{ github.workspace }}
+          AIRSTACK_TESTED_SHA: ${{ github.sha }}
+          AIRSTACK_PR_NUMBER: ${{ github.event.pull_request.number }}
         run: pytest tests/ -m unit
 
       - name: Upload unit-test results