The Agent Patch Loosened the Assertion and the Suite Stayed Green: An Oracle-Latitude Gate

-    assert invoice.total == Decimal("42.00")
+    assert invoice.total > 0

# latitude.py  (Python 3.11+, stdlib only)
from __future__ import annotations

import argparse
import ast
import json
from pathlib import Path

CMP_RANK = {
    ast.Eq: 0, ast.NotEq: 0, ast.Is: 0, ast.IsNot: 0,
    ast.Lt: 1, ast.LtE: 1, ast.Gt: 1, ast.GtE: 1,
    ast.In: 2, ast.NotIn: 2,
}
UNIT_MATCHERS = {
    "assertEqual": 0, "assertIs": 0, "assertIsNone": 0,
    "assertIn": 2, "assertAlmostEqual": 2, "assertTrue": 3,
}
EXC_RANK = {"ValueError": 0, "KeyError": 0, "TypeError": 0,
            "Exception": 3, "BaseException": 3}


def test_functions(tree: ast.AST):
    for node in ast.walk(tree):
        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) \
                and node.name.startswith("test"):
            yield node


def tolerance(node: ast.AST) -> float | None:
    """Recognise abs(a - b) < X and pytest.approx(..., abs=X|rel=X)."""
    if isinstance(node, ast.Compare) and len(node.ops) == 1:
        left = node.left
        right = node.comparators[0]
        if (isinstance(left, ast.Call) and getattr(left.func, "id", None) == "abs"
                and isinstance(right, ast.Constant)
                and isinstance(right.value, (int, float))):
            return float(right.value)
    if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "approx":
        for kw in node.keywords:
            if kw.arg in ("abs", "rel") and isinstance(kw.value, ast.Constant):
                return float(kw.value.value)
    return None


def facts(fn: ast.AST) -> dict:
    out = {"compare": [], "tolerance": [], "raises": [],
           "asserts": 0, "skip": False}
    for node in ast.walk(fn):
        if isinstance(node, ast.Assert):
            out["asserts"] += 1
            if isinstance(node.test, ast.Compare):
                out["compare"].append(
                    sorted(CMP_RANK.get(type(op), 2) for op in node.test.ops))
            else:
                out["compare"].append([3])  # bare truthiness
        if isinstance(node, ast.Call):
            name = getattr(node.func, "attr", None)
            if name in UNIT_MATCHERS:
                out["asserts"] += 1
                out["compare"].append([UNIT_MATCHERS[name]])
            if name == "raises" and node.args and isinstance(node.args[0], ast.Name):
                out["raises"].append(EXC_RANK.get(node.args[0].id, 1))
        tol = tolerance(node)
        if tol is not None:
            out["tolerance"].append(tol)
        if isinstance(node, ast.Attribute) and node.attr in ("skip", "xfail"):
            out["skip"] = True
    return out


def scan(root: Path) -> dict[str, dict]:
    found = {}
    for path in sorted(root.rglob("test_*.py")):
        tree = ast.parse(path.read_text(encoding="utf-8"))
        for fn in test_functions(tree):
            found[f"{path.relative_to(root)}::{fn.name}"] = facts(fn)
    return found

SIGNAL_WEIGHTS = {"compare": "rank delta", "raises": "rank delta",
                  "tolerance": 2, "dropped_assert": 1, "new_skip": 4}


def score(base: dict, patch: dict) -> tuple[int, list[str]]:
    total, why = 0, []
    for i, ranks in enumerate(patch["compare"]):
        before = base["compare"][i] if i < len(base["compare"]) else [0]
        for old, new in zip(before, ranks):
            if new > old:
                total += new - old
                why.append(f"compare rank {old}->{new}")
    for old, new in zip(base["raises"], patch["raises"]):
        if new > old:
            total += new - old
            why.append(f"exception rank {old}->{new}")
    for old, new in zip(base["tolerance"], patch["tolerance"]):
        if new > old:
            total += SIGNAL_WEIGHTS["tolerance"]
            why.append(f"tolerance {old}->{new}")
    dropped = base["asserts"] - patch["asserts"]
    if dropped > 0:
        total += dropped * SIGNAL_WEIGHTS["dropped_assert"]
        why.append(f"{dropped} assertion(s) removed")
    if patch["skip"] and not base["skip"]:
        total += SIGNAL_WEIGHTS["new_skip"]
        why.append("skip/xfail added")
    return total, why


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("base")
    ap.add_argument("patched")
    ap.add_argument("--max-score", type=int, default=4)
    ap.add_argument("--json")
    args = ap.parse_args(argv)

    base, patched = scan(Path(args.base)), scan(Path(args.patched))
    report, total_score = {}, 0
    for name in sorted(set(base) & set(patched)):
        value, why = score(base[name], patched[name])
        if value:
            total_score += value
            report[name] = {"score": value, "signals": why}

    verdict = "FAIL" if total_score >= args.max_score else "PASS"
    print(json.dumps({"verdict": verdict, "score": total_score,
                      "tests": report}, indent=2))
    if args.json:
        Path(args.json).write_text(json.dumps(report, indent=2), encoding="utf-8")
    return 0 if verdict == "PASS" else 1


if __name__ == "__main__":
    raise SystemExit(main())

git worktree add /tmp/base origin/main
python latitude.py /tmp/base/src/tests src/tests --max-score 4 --json latitude.json