Regression Testing

# Regression test runner for agent changes
class RegressionTestRunner:
    def __init__(self, agent_fn, eval_dataset: list):
        self.agent_fn  = agent_fn
        self.eval_data = eval_dataset

    def run(self) -> dict:
        results = {"passed": 0, "failed": 0, "regressions": []}

        for case in self.eval_data:
            # Run the agent (simulated here)
            agent_result = self.agent_fn(case["input"])

            # Check outcome
            outcome_match = agent_result.get("outcome") == case["expected_outcome"]
            escalation_match = (
                agent_result.get("escalated", False) == case["should_escalate"]
            )

            passed = outcome_match and escalation_match

            if passed:
                results["passed"] += 1
                print(f"  ✓ {case['id']} ({case['category']})")
            else:
                results["failed"] += 1
                results["regressions"].append({
                    "case_id":  case["id"],
                    "expected": case["expected_outcome"],
                    "got":      agent_result.get("outcome"),
                })
                print(f"  ✗ {case['id']} ({case['category']}) — "
                      f"expected '{case['expected_outcome']}', got '{agent_result.get('outcome')}'")

        total = len(self.eval_data)
        results["pass_rate"] = round(results["passed"] / total * 100, 1)
        results["total"]     = total
        return results

# Mock agent (simulates a new version that regressed on case 003)
def mock_agent_v2(input_data):
    oid = input_data["order_id"]
    if oid == "ORD-050":  return {"outcome": "rejected", "escalated": False}
    if oid == "ORD-100":  return {"outcome": "success",  "escalated": False}
    if oid == "ORD-999":  return {"outcome": "success",  "escalated": False}  # Regression!
    return {"outcome": "failed", "escalated": False}

runner = RegressionTestRunner(mock_agent_v2, EVAL_DATASET)
results = runner.run()
print(f"\nPass rate: {results['pass_rate']}% ({results['passed']}/{results['total']})")
if results["regressions"]:
    print("REGRESSIONS DETECTED — do not deploy this version!")