diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..aeb822d --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,126 @@ +name: Benchmarks + +on: + pull_request: + paths: + - .github/workflows/benchmarks.yml + - benchmarks/** + - src/** + - Directory.Build.props + - OpenDisNet.slnx + push: + branches: [main] + paths: + - .github/workflows/benchmarks.yml + - benchmarks/** + - src/** + - Directory.Build.props + - OpenDisNet.slnx + schedule: + - cron: "17 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: Benchmark harness smoke test + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: | + 8.x + 9.x + 10.x + - run: dotnet restore --locked-mode + - name: Build benchmark harness + run: dotnet build benchmarks/OpenDisNet.Benchmarks/OpenDisNet.Benchmarks.csproj --configuration Release --framework net10.0 --no-restore --no-incremental + - name: Exercise every benchmark and runtime once + run: >- + dotnet run + --project benchmarks/OpenDisNet.Benchmarks/OpenDisNet.Benchmarks.csproj + --configuration Release + --framework net10.0 + --no-build + --no-restore + -- + --filter '*' + --job Dry + --runtimes net8.0 net9.0 net10.0 + --join + --exporters JSON + --artifacts artifacts/benchmarks + - name: Add diagnostic report to job summary + shell: bash + run: | + report="$(find artifacts/benchmarks/results -name '*-report-github.md' -print -quit)" + { + echo '## Benchmark harness smoke test' + echo + echo '> BenchmarkDotNet Dry mode validates that every benchmark builds and runs. Its timings are diagnostic only and are not performance measurements.' + echo + cat "$report" + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload diagnostic results + uses: actions/upload-artifact@v7 + with: + name: benchmark-smoke-${{ github.run_id }} + path: artifacts/benchmarks/** + retention-days: 7 + + benchmark: + name: Full cross-runtime benchmarks + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: | + 8.x + 9.x + 10.x + - run: dotnet restore --locked-mode + - name: Build benchmark harness + run: dotnet build benchmarks/OpenDisNet.Benchmarks/OpenDisNet.Benchmarks.csproj --configuration Release --framework net10.0 --no-restore --no-incremental + - name: Measure all supported runtimes + run: >- + dotnet run + --project benchmarks/OpenDisNet.Benchmarks/OpenDisNet.Benchmarks.csproj + --configuration Release + --framework net10.0 + --no-build + --no-restore + -- + --filter '*' + --runtimes net8.0 net9.0 net10.0 + --join + --exporters JSON + --artifacts artifacts/benchmarks + - name: Add results to job summary + shell: bash + run: | + report="$(find artifacts/benchmarks/results -name '*-report-github.md' -print -quit)" + { + echo '## Cross-runtime benchmark results' + echo + echo '> GitHub-hosted runner results are useful for comparisons within this run. Use controlled, dedicated hardware for strict historical regression decisions.' + echo + cat "$report" + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload benchmark results + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ github.run_id }} + path: artifacts/benchmarks/** + retention-days: 30 diff --git a/README.md b/README.md index 25e9f55..a2c26b1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # OpenDisNet [![CI](https://github.com/RejectKid/OpenDisNet/actions/workflows/ci.yml/badge.svg)](https://github.com/RejectKid/OpenDisNet/actions/workflows/ci.yml) +[![Benchmarks](https://github.com/RejectKid/OpenDisNet/actions/workflows/benchmarks.yml/badge.svg)](https://github.com/RejectKid/OpenDisNet/actions/workflows/benchmarks.yml) [![NuGet](https://img.shields.io/nuget/v/OpenDisNet.svg)](https://www.nuget.org/packages/OpenDisNet) [![GitHub Release](https://img.shields.io/github/v/release/RejectKid/OpenDisNet)](https://github.com/RejectKid/OpenDisNet/releases/latest) @@ -145,6 +146,18 @@ dotnet run --project benchmarks/OpenDisNet.Benchmarks -c Release -f net10.0 Replace `net10.0` with `net8.0` or `net9.0` for runtime comparisons. Benchmark artifacts are written beneath `BenchmarkDotNet.Artifacts` and are not committed. +The [benchmark workflow](https://github.com/RejectKid/OpenDisNet/actions/workflows/benchmarks.yml) +runs a fast BenchmarkDotNet `Dry` smoke test across .NET 8, 9, and 10 for +relevant pull requests. Dry-mode timings only validate the harness; they are not +performance measurements. Full cross-runtime measurements run after relevant +changes land on `main`, every Monday, and on manual dispatch. The workflow adds +the Markdown table to its job summary and retains the HTML, Markdown, CSV, and +JSON results as downloadable artifacts for 30 days. + +GitHub-hosted runners are appropriate for comparing runtimes within one run. +Use controlled, dedicated hardware before treating results from different runs +as a strict performance regression gate. + ## Security Treat network datagrams as untrusted. See [`SECURITY.md`](SECURITY.md) for the