diff --git a/exploitation/llmsectest/.gitignore b/exploitation/llmsectest/.gitignore new file mode 100644 index 0000000..a2309c5 --- /dev/null +++ b/exploitation/llmsectest/.gitignore @@ -0,0 +1,13 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +.venv/ +.env + +# Logs +*.log +reports/*.sarif +reports/*.html +reports/*.json +reports/*.md diff --git a/exploitation/llmsectest/Makefile b/exploitation/llmsectest/Makefile new file mode 100644 index 0000000..683afb9 --- /dev/null +++ b/exploitation/llmsectest/Makefile @@ -0,0 +1,52 @@ +SANDBOX_NAME := $(shell uv run python -c 'import tomllib, pathlib; print(tomllib.loads(pathlib.Path("config/config.toml").read_text())["target"]["sandbox"])') +SANDBOX_DIR := ../../sandboxes/$(SANDBOX_NAME) + +.PHONY: help setup attack stop all sync lock format + +# Default target +help: + @echo "LLMSecTest Exploitation - Available Commands:" + @echo "" + @echo " make setup - Build and start the local LLM sandbox" + @echo " make attack - Run an LLMSecTest scan against the sandbox" + @echo " make stop - Stop and remove the sandbox container" + @echo " make all - Run setup, attack, and stop in sequence" + @echo " make sync - Sync dependencies with uv" + @echo " make lock - Lock dependencies with uv" + @echo " make format - Format code with black, isort, and mypy" + @echo "" + @echo "Environment:" + @echo " - Sandbox Directory: $(SANDBOX_DIR)" + @echo "" + +sync: + uv sync + +lock: + uv lock + +format: + uv run black . + uv run isort . + uv run mypy . + +setup: + @echo "๐Ÿš€ Setting up Red Team environment..." + $(MAKE) -C $(SANDBOX_DIR) run-gradio-headless + @echo "โณ Waiting for service to be ready..." + @sleep 5 + @echo "โœ… Environment ready!" + +attack: sync + @echo "๐Ÿ” Running LLMSecTest scan..." + @mkdir -p reports + uv run python attack.py + +stop: + @echo "๐Ÿงน Tearing down Red Team environment..." + $(MAKE) -C $(SANDBOX_DIR) stop-gradio + $(MAKE) -C $(SANDBOX_DIR) down + @echo "โœ… Environment cleaned up!" + +all: stop setup attack stop + @echo "LLMSecTest Exploitation - Completed!" diff --git a/exploitation/llmsectest/README.md b/exploitation/llmsectest/README.md new file mode 100644 index 0000000..8bd990e --- /dev/null +++ b/exploitation/llmsectest/README.md @@ -0,0 +1,164 @@ +# Red Team Example: LLMSecTest on the LLM Sandbox + +This directory runs **[LLMSecTest](https://github.com/wehnsdaefflae/llmsectest)** against the +`llm_local` sandbox. LLMSecTest is an MIT-licensed scanner for the OWASP Top 10 for LLM +Applications that ships as a **pytest plugin**: a scan is a test run, a failing security test +is a CVSS v4.0 scored OWASP finding, and the output is **SARIF** that a CI code-scanning tab +reads without conversion. + +Two things make it complementary to the scanners already in `exploitation/`: + +* it emits **SARIF** with the OWASP taxonomy, so findings land in GitHub code scanning + alongside the rest of a pipeline's results; +* `--repo` adds a **white-box** LLM03 supply-chain scan of the sandbox's own dependency + manifests, so this example covers a category a purely black-box scanner cannot reach. + +--- + +## Table of Contents + +1. [Attack Strategy](#attack-strategy) +2. [Prerequisites](#prerequisites) +3. [Running the Sandbox](#running-the-sandbox) +4. [Configuration](#configuration) +5. [Attack Workflow](#attack-workflow) +6. [Cleaning Up](#cleaning-up) +7. [Files Overview](#files-overview) +8. [OWASP Top 10 Coverage](#owasp-top-10-coverage) +9. [Two notes about this sandbox](#two-notes-about-this-sandbox) + +--- + +## Attack Strategy + +```mermaid +graph LR + subgraph "Attacker Environment (Local)" + AttackScript[attack.py] + Config[config/config.toml] + Reports[Reports
SARIF + HTML] + end + + subgraph "LLMSecTest" + Suite[pytest probe suite
LLM01-LLM10] + Repo[--repo
LLM03 manifest scan] + end + + subgraph "Target Sandbox (Container)" + MockAPI[OpenAI mock
FastAPI :8000] + end + + subgraph "LLM Backend (Local Host)" + Ollama[Ollama Server
:11434] + end + + Config --> AttackScript + AttackScript --> Suite + AttackScript --> Repo + Suite -->|POST /v1/chat/completions| MockAPI + MockAPI --> Ollama + Repo -->|reads manifests| MockAPI + Suite --> Reports + Repo --> Reports +``` + +## Prerequisites + +* [`uv`](https://docs.astral.sh/uv/), Podman, and a running Ollama on port `11434` + (the same prerequisites as `exploitation/garak`) +* the model named in `config/config.toml`, pulled into Ollama + (`make -C ../../sandboxes/llm_local ollama-pull`) + +## Running the Sandbox + +```bash +make setup +``` + +Starts `sandboxes/llm_local`, which exposes the OpenAI-compatible mock on `:8000`. + +## Configuration + +`config/config.toml`: + +```toml +[target] +sandbox = "llm_local" + +[endpoint] +base_url = "http://localhost:8000/v1" +api_key = "sk-mock-key" +model = "gpt-oss:20b" +``` + +`attack.py` puts `base_url` and `api_key` into `OPENAI_BASE_URL` and `OPENAI_API_KEY` and +calls the CLI, so the run is reproducible by hand: + +```bash +OPENAI_BASE_URL=http://localhost:8000/v1 OPENAI_API_KEY=sk-mock-key \ + llmsectest --target openai:gpt-oss:20b \ + --repo ../../sandboxes/llm_local \ + --sarif-output reports/GenAI-Red-Team.sarif \ + --render-sarif reports/GenAI-Red-Team.html +``` + +## Attack Workflow + +```bash +make attack +``` + +**A non-zero exit means findings.** That is the expected outcome against a sandbox built to be +attacked, so treat it as the result rather than as a failed run. A probe LLMSecTest could not get an answer out of is recorded +**inconclusive** and never scored as a finding, so a broken target and a vulnerable one do +not look alike. + +## Cleaning Up + +```bash +make stop +``` + +## Files Overview + +| File | Purpose | +|---|---| +| `attack.py` | sets the endpoint env vars and runs the CLI | +| `config/config.toml` | sandbox name, base URL, key, model | +| `Makefile` | `setup` / `attack` / `stop` / `all`, same shape as the other examples | +| `pyproject.toml` | pins `llmsectest[openai]` | +| `reports/` | SARIF and rendered HTML land here | + +## OWASP Top 10 Coverage + +Against this sandbox the run exercises **8 of 10** categories. LLMSecTest prints the map at +the end of every scan and names what it did **not** exercise and why, so a partial run never +reads as a full one. + +| Category | How it is covered here | +|---|---| +| LLM01 Prompt Injection | marker-injection corpus plus built-in jailbreak prompts (`--redteam-set` runs the full JailbreakBench set) | +| LLM02 Sensitive Information Disclosure | four disclosure mechanisms against a seeded secret | +| LLM03 Supply Chain | `--repo` scans the sandbox's manifests; add `--osv` for known CVEs | +| LLM04 Data and Model Poisoning | not exercised here; needs `--model-scan ` over serialized model files | +| LLM05 Improper Output Handling | payloads checked for surviving unescaped into the response | +| LLM06 Excessive Agency | forged-authorization probes | +| LLM07 System Prompt Leakage | instruction-restatement corpus with de-obfuscation of encoded leaks | +| LLM08 Vector and Embedding Weaknesses | not exercised here; needs a RAG target, see below | +| LLM09 Misinformation | fabrication probes | +| LLM10 Unbounded Consumption | bounded resource-exhaustion probes | + +**Natural next step:** point the same tool at `sandboxes/RAG_local` with +`--target app: --app-canary ` and +`--app-rag-poison ` to cover LLM08 as well. + +## Two notes about this sandbox + +Both were found by running it. Both would trip up the next person. + +1. **The mock does not serve `GET /v1/models`.** It serves `POST /v1/chat/completions` and + `GET /health` only. So `llmsectest --preflight`, which health-checks a local server + through the models endpoint, reports a perfectly healthy sandbox as unreachable. It is + deliberately not used here. +2. **The mock enforces `Authorization: Bearer sk-mock-key`.** The key is not optional even + though the backend is local, so `OPENAI_API_KEY` has to be set to exactly that value. diff --git a/exploitation/llmsectest/attack.py b/exploitation/llmsectest/attack.py new file mode 100644 index 0000000..47efb95 --- /dev/null +++ b/exploitation/llmsectest/attack.py @@ -0,0 +1,63 @@ +"""Run an LLMSecTest scan against the llm_local sandbox. + +LLMSecTest is a pytest plugin. A scan is a test run. A failing security test is rendered as a +CVSS v4.0 scored OWASP finding in SARIF. This script sets the two +environment variables the OpenAI-compatible client reads and shells out to the CLI, so +what runs here is exactly what a reader can run by hand. + +Two notes about this sandbox in particular, both found by running it: + +* the mock serves ``POST /v1/chat/completions`` and ``GET /health`` and **not** + ``GET /v1/models``, so ``llmsectest --preflight`` is deliberately not used here. It + would report a healthy sandbox as unreachable. +* the mock enforces ``Authorization: Bearer sk-mock-key``, so the key is not optional + even though the backend is local. + +``--repo`` adds the white-box LLM03 supply-chain scan over the sandbox's own dependency +manifests, which is why this run covers a category a black-box scanner cannot reach. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess +import sys +import tomllib + +HERE = pathlib.Path(__file__).parent +CONFIG = tomllib.loads((HERE / "config" / "config.toml").read_text()) +REPORTS = HERE / "reports" +SANDBOX = HERE.parent.parent / "sandboxes" / CONFIG["target"]["sandbox"] + + +def main() -> int: + REPORTS.mkdir(exist_ok=True) + endpoint = CONFIG["endpoint"] + + env = dict(os.environ) + env["OPENAI_BASE_URL"] = endpoint["base_url"] + env["OPENAI_API_KEY"] = endpoint["api_key"] + + cmd = [ + sys.executable, "-m", "llmsectest", + "--target", f"openai:{endpoint['model']}", + "--repo", str(SANDBOX), + "--sarif-output", str(REPORTS / "GenAI-Red-Team.sarif"), + "--render-sarif", str(REPORTS / "GenAI-Red-Team.html"), + "-q", + ] + print("running:", " ".join(cmd)) + completed = subprocess.run(cmd, env=env, cwd=HERE) + + # A non-zero exit means the target was vulnerable. That is the expected outcome against + # a sandbox built to be attacked. Only a crash is a failure of the run. LLMSecTest keeps + # the two apart: a probe it could not get an answer out of is recorded inconclusive and + # is never scored as a finding. + print(f"\nscan exit={completed.returncode}; non-zero means findings were reported") + print(f"reports: {REPORTS}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/exploitation/llmsectest/config/config.toml b/exploitation/llmsectest/config/config.toml new file mode 100644 index 0000000..26c2f7a --- /dev/null +++ b/exploitation/llmsectest/config/config.toml @@ -0,0 +1,9 @@ +[target] +sandbox = "llm_local" + +# The sandbox's OpenAI-compatible mock. It enforces `Authorization: Bearer sk-mock-key` +# (sandboxes/llm_local/app/mocks/openai.py) and serves POST /v1/chat/completions only. +[endpoint] +base_url = "http://localhost:8000/v1" +api_key = "sk-mock-key" +model = "gpt-oss:20b" diff --git a/exploitation/llmsectest/pyproject.toml b/exploitation/llmsectest/pyproject.toml new file mode 100644 index 0000000..a582fe6 --- /dev/null +++ b/exploitation/llmsectest/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "llmsectest-exploitation" +version = "0.1.0" +description = "LLMSecTest exploitation setup for the GenAI Red Team Lab" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "llmsectest[openai]>=0.2.0", +] + +[dependency-groups] +dev = [ + "black>=24.0.0", + "isort>=5.0.0", + "mypy>=1.0.0", +] + +[tool.mypy] +ignore_missing_imports = true diff --git a/exploitation/llmsectest/reports/.gitkeep b/exploitation/llmsectest/reports/.gitkeep new file mode 100644 index 0000000..e69de29