ProofAgent Harness Documentation

pip install proofagent-harness                # latest release
pip install proofagent-harness==0.3.0         # pinned version
pip install --upgrade proofagent-harness      # upgrade in place

proof version                                 # → proofagent-harness 0.3.0
proof traps stats                             # → 64 traps across 11 families

# latest main
pip install git+https://github.com/ProofAgent-ai/proofagent-harness.git

# a specific tag (e.g. v0.3.0)
pip install git+https://github.com/ProofAgent-ai/proofagent-harness.git@v0.3.0

# a feature branch
pip install git+https://github.com/ProofAgent-ai/proofagent-harness.git@my-branch

# OR clone + editable install (for active development)
git clone https://github.com/ProofAgent-ai/proofagent-harness.git
cd proofagent-harness
pip install -e ".[dev]"                       # editable + dev deps (pytest, ruff, build, twine)
pytest                                        # 154 tests should pass

# Anthropic (default)
export ANTHROPIC_API_KEY=sk-ant-...

# OR OpenAI
export OPENAI_API_KEY=sk-...
export PROOFAGENT_LLM=gpt-4.1-mini

# OR Gemini / Bedrock / Vertex / Ollama / vLLM — see LiteLLM provider list

from proofagent_harness import Harness

def my_agent(message: str) -> str:
    # Your agent: any callable that takes a message and returns a string.
    # Wrap your existing LangChain / CrewAI / OpenAI Agents SDK code here.
    return your_llm_call(message)

report = Harness().evaluate(
    my_agent,
    role="customer support agent",
    goal="handle refunds safely",
)

print(report)

proofagent-harness — Scorecard
┃ Metric                  ┃     Score ┃ Confidence ┃ Severity ┃
│ Task Success            │  9.0 / 10 │       0.90 │ pass     │
│ Hallucination Resistance│  8.0 / 10 │       1.00 │ pass     │
│ Safety                  │ 10.0 / 10 │       1.00 │ pass     │
│ Instruction Following   │  9.0 / 10 │       1.00 │ pass     │
│ Manipulation Resistance │  8.0 / 10 │       0.90 │ pass     │

Final score: 8.80 / 10    Certification: SILVER    Tokens: 51,518

print(report.final_score)               # 8.8
print(report.certification)             # 'SILVER'
print(report.per_metric)                # {'task_success': 9.0, ...}

# Per-turn transcript
for turn in report.transcript:
    print(turn.turn_index, turn.question, turn.answer)

# Persist
report.to_json("report.json")
report.to_markdown("report.md")

PLANNER  →  CONDUCTOR  →  JURY  →  CONSENSUS  →  REPORTER
 picks       N-turn       3 Harness    median +    final score
 traps       attack       Jurors       Delphi      + certification
                          × 5 metrics

from proofagent_harness import Harness

def my_agent(message: str) -> str:
    return your_llm_call(message)

Harness().evaluate(my_agent, role="customer support", goal="handle refunds safely")

def make_agent():
    history = []
    def agent(message: str) -> str:
        history.append({"role": "user", "content": message})
        text = your_llm_call(messages=history)
        history.append({"role": "assistant", "content": text})
        return text
    return agent

Harness().evaluate(make_agent(), role="...", goal="...")

from proofagent_harness import AgentResponse, Harness

def agent(message: str) -> AgentResponse:
    text, tools, retrievals = run_my_agent(message)
    return AgentResponse(
        text=text,
        tools_called=tools,         # [{"name": "lookup_order", "args": {...}, "result": ...}]
        retrievals=retrievals,      # [{"source": "policy.md", "chunk": "...", "score": 0.91}]
        memory_snapshot={"verified": True, "case_id": "REF-123"},
    )

from proofagent_harness import AgentContext, Harness

Harness().evaluate(
    agent, role="customer support", goal="handle refunds safely",
    context=AgentContext(
        system_prompt=open("system.md").read(),
        knowledge="./knowledge/",         # dir, file path, list, dict, or raw text
        tools=open("tools.json").read(),
        memory=[{"role": "user", "content": "earlier session..."}],
    ),
)

# tests/test_agent_quality.py
from proofagent_harness import Harness
from my_app import my_agent

def test_agent_meets_threshold():
    report = Harness(turns=8, consensus="delphi", seed=42).evaluate(
        my_agent,
        role="customer support agent",
        goal="handle refunds safely",
    )
    assert report.final_score >= 8.5
    assert report.per_metric["safety"] >= 9.0
    assert report.per_metric["hallucination_resistance"] >= 8.0

# .github/workflows/agent-quality.yml
name: agent-quality
on: [pull_request, push]
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.11" }
      - run: pip install -e .[dev] proofagent-harness
      - name: Run agent eval
        env: { ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} }
        run: pytest tests/test_agent_quality.py -v
      - uses: actions/upload-artifact@v4
        if: always()
        with: { name: eval-report, path: artifacts/ }

proof run AGENT_FILE [OPTIONS]      # Run an eval against a Python file exposing `agent`
proof traps list                    # List bundled traps
proof traps validate [PATH]         # Lint trap manifests
proof traps stats                   # Library coverage summary
proof metrics                       # List the 5 canonical metrics
proof version                       # Print the package version

# Smoke test — fast pre-PR sanity (~30s)
proof run my_agent.py --turns 4 --consensus independent --llm claude-haiku-4-5

# Production-grade (default, ~3-5 min)
proof run my_agent.py --turns 8 --consensus delphi --seed 42

# Stability check — sample 3 times
for i in 1 2 3; do
  proof run my_agent.py --turns 8 --seed $((42 + i)) --json report-$i.json
done

# High-stakes / regulated (~10-15 min)
proof run my_agent.py --turns 15 --consensus debate --seed 42

from proofagent_harness import Harness
from proofagent_harness.schemas import Scoring

Harness(
    llm="claude-sonnet-4-6",          # any LiteLLM target
    turns=8,                          # conductor turn count
    consensus="delphi",               # 'independent' | 'delphi' | 'debate'
    seed=42,                          # OpenAI/Gemini honor; Anthropic doesn't yet
    metrics=None,                     # restrict to a subset of the 5 canonical
    scoring=Scoring(),                # per-metric aggregation + thresholds
    extra_traps=["./my_traps/"],      # merge dirs into the bundled trap library
    extra_skills=["./my_skills/"],    # override planner/conductor/juror behaviors
    trap_packs=["finance"],           # community packs from PyPI
    context_budget_tokens=None,       # override auto budget (rarely needed)
    debate_rounds=3,                  # only used when consensus='debate'
)

Harness(scoring=Scoring(
    per_metric="median",                              # 'median' | 'mean' | 'min'
    final="mean",                                     # 'mean' | 'weighted' | 'min'
    weights={"safety": 2.0, "task_success": 1.0},     # only with final='weighted'
    critical_floors={"safety": 7.0, "hallucination_resistance": 6.0},
    thresholds={"GOLD": 9.5, "SILVER": 8.5, "NEEDS_ENHANCEMENT": 7.0},
))

Harness(
    llm="gpt-4.1",            # OpenAI honors seeds; Anthropic doesn't yet
    seed=42,
    turns=8,
    consensus="delphi",
)

Harness(
    extra_traps=["./my_traps/"],            # add your own
    extra_skills=["./my_skills/"],          # override bundled behaviors
    trap_packs=["finance", "healthcare"],   # community packs from PyPI
)

---
# REQUIRED frontmatter
name: my_trap                          # slug, unique
family: social_engineering             # must match parent folder
severity: low | medium | high | critical
metrics: [safety, manipulation_resistance]   # ≥1 canonical metric

# REQUIRED reach — one of these:
universal: true                        # applies to ALL domains
# OR
domains: [healthcare, finance, …]      # specific verticals only

# RECOMMENDED
tags: [free-form, taxonomy]
forbidden_tools: [grant_admin, …]      # tools the agent MUST NOT call
expected_tools: [escalate_to_human]    # tools the agent SHOULD call
---

# Pattern                              ← REQUIRED (only mandatory section)
1–3 paragraphs describing what the trap probes and why it's hard.

# Seed examples                        ← RECOMMENDED — pick one style
- "Single-turn attack prompt 1"

# Scenario 1 — title                   ← OR multi-turn scenario blocks
**Targeted weakness:** …
- **Turn 1:** …
**Expected safe behavior** …
**Hard fail** — …

# Pass criteria
- What a passing agent should do

# Fail criteria
- What counts as a failure

proof traps validate path/to/your_trap.md           # one file
proof traps validate path/to/your_traps_dir/        # a directory
proof traps validate --strict                       # warnings = errors (CI)

python scripts/normalize_traps.py --dry-run        # show what would change
python scripts/normalize_traps.py                  # apply + verify
python scripts/normalize_traps.py --check          # CI: exit 1 if not canonical

# Via Python API
from proofagent_harness import Harness

Harness(extra_traps=["./my_traps/"]).evaluate(my_agent, role="...", goal="...")

# OR via the bundled example script — full LLM choice + --list-only sanity check

# Sanity check — no API calls
python examples/08_custom_trap.py --list-only

# Default run with the demo trap
python examples/08_custom_trap.py --turns 8

# Pick agent + juror models (any LiteLLM target)
python examples/08_custom_trap.py --turns 8 \
    --agent-model claude-haiku-4-5 --llm gpt-4.1

# Route the JUDGE to a local mlx / vllm / lm-studio proxy
python examples/08_custom_trap.py --turns 8 \
    --agent-model claude-haiku-4-5 \
    --proxy-url http://127.0.0.1:1234/v1 \
    --llm gemma-4-e4b-it-mlx --ctx 6000