Keep the Score Contract Out of the Agent's Write Set

#!/usr/bin/env bash
# record_parent_contract.sh — proposal
set -euo pipefail
PARENT="$(git rev-parse HEAD)"
python tools/score_lib.py record --parent "$PARENT" --out score_contract.json

#!/usr/bin/env bash
# deny_score_paths.sh — proposal
set -euo pipefail
: "${PARENT_SHA:?}"
DENY_REGEX='^(score_contract\.json|tools/score_lib\.py|tests/fixtures/|pytest\.ini|pyproject\.toml|conftest\.py|.*/conftest\.py)$'
changed="$(git diff --name-only "$PARENT_SHA"...HEAD)"
if echo "$changed" | grep -E "$DENY_REGEX"; then
  echo "score surface is in the agent write set" >&2
  exit 1
fi

# tools/score_lib.py — proposal (helpers)
from __future__ import annotations

import hashlib
from pathlib import Path

CONFIG_CANDIDATES = ("pytest.ini", "pyproject.toml", "conftest.py")


def sha256_file(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def hash_tree(root: Path) -> dict[str, str]:
    out: dict[str, str] = {}
    if not root.exists():
        return out
    for path in sorted(p for p in root.rglob("*") if p.is_file()):
        out[path.as_posix()] = sha256_file(path)
    return out


def hash_runner_config(repo: Path) -> dict[str, str]:
    out: dict[str, str] = {}
    for name in CONFIG_CANDIDATES:
        path = repo / name
        if path.is_file():
            out[path.as_posix()] = sha256_file(path)
    for path in sorted(repo.rglob("conftest.py")):
        out[path.as_posix()] = sha256_file(path)
    return out

{
  "parent_sha": "REPLACE_WITH_PARENT_SHA",
  "fixture_root": "tests/fixtures",
  "fixture_hashes": {},
  "runner_config_hashes": {},
  "property_budget": {
    "max_examples": 100,
    "deadline_seconds": 30
  },
  "assert_floor": 0,
  "collect_floor": 0,
  "flake_leases": [
    {
      "nodeid": "tests/test_retry.py::test_backoff_jitter",
      "reason": "upstream 503 body changed; cassette refresh pending",
      "expires_at": "2026-09-25T00:00:00Z"
    }
  ]
}

# tools/score_lib.py — proposal (floors)
import ast
import json
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

ASSERT_ATTRS = {
    "assertEqual",
    "assertTrue",
    "assertFalse",
    "assertRaises",
    "assertAlmostEqual",
}


def count_asserts(tree: Path) -> int:
    total = 0
    for path in tree.rglob("test_*.py"):
        node = ast.parse(path.read_text(encoding="utf-8"))
        for item in ast.walk(node):
            if isinstance(item, ast.Assert):
                total += 1
            elif isinstance(item, ast.Call) and isinstance(item.func, ast.Attribute):
                if item.func.attr in ASSERT_ATTRS:
                    total += 1
    return total


def collect_count() -> int:
    proc = subprocess.run(
        [sys.executable, "-m", "pytest", "--collect-only", "-q"],
        check=True,
        capture_output=True,
        text=True,
    )
    return sum(1 for ln in proc.stdout.splitlines() if "::" in ln)


def record_contract(parent: str, out: Path) -> None:
    contract = {
        "parent_sha": parent,
        "recorded_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "fixture_root": "tests/fixtures",
        "fixture_hashes": hash_tree(Path("tests/fixtures")),
        "runner_config_hashes": hash_runner_config(Path(".")),
        "property_budget": {"max_examples": 100, "deadline_seconds": 30},
        "assert_floor": count_asserts(Path("tests")),
        "collect_floor": collect_count(),
        "flake_leases": [],
    }
    out.write_text(json.dumps(contract, indent=2) + "\n", encoding="utf-8")

# tools/score_lib.py — proposal (score)
from datetime import datetime, timezone


def active_skips(leases: list[dict]) -> list[str]:
    now = datetime.now(timezone.utc)
    skip: list[str] = []
    for lease in leases:
        expires = datetime.fromisoformat(lease["expires_at"].replace("Z", "+00:00"))
        if expires <= now:
            raise SystemExit(f"expired flake lease: {lease['nodeid']}")
        skip.append(lease["nodeid"])
    return skip


def score() -> int:
    contract = json.loads(Path("score_contract.json").read_text(encoding="utf-8"))
    if hash_tree(Path(contract["fixture_root"])) != contract["fixture_hashes"]:
        raise SystemExit("fixture hash mismatch")
    if hash_runner_config(Path(".")) != contract["runner_config_hashes"]:
        raise SystemExit("runner config hash mismatch")
    if count_asserts(Path("tests")) < contract["assert_floor"]:
        raise SystemExit("assertion floor missed")
    if collect_count() < contract["collect_floor"]:
        raise SystemExit("collect floor missed")

    budget = contract["property_budget"]
    cmd = [
        sys.executable, "-m", "pytest", "-q", "--maxfail=1",
        f"--timeout={budget['deadline_seconds']}",
    ]
    for nodeid in active_skips(contract["flake_leases"]):
        cmd.extend(["--deselect", nodeid])
    # Optional: export HYPOTHESIS_MAX_EXAMPLES from budget["max_examples"]
    return subprocess.call(cmd)

def main() -> None:
    if sys.argv[1] == "record":
        parent = sys.argv[sys.argv.index("--parent") + 1]
        out = Path(sys.argv[sys.argv.index("--out") + 1])
        record_contract(parent, out)
        return
    if sys.argv[1] == "score":
        raise SystemExit(score())
    raise SystemExit("usage: score_lib.py record|score")


if __name__ == "__main__":
    main()