Reliability Testing

# Fault injection testing for agent reliability
import random

class FaultInjector:
    """Wrap a tool to inject faults for testing."""
    def __init__(self, tool_fn, fault_rate: float = 0.3,
                 fault_type: str = "timeout"):
        self.tool_fn = tool_fn
        self.fault_rate = fault_rate
        self.fault_type = fault_type
        self.call_count = 0
        self.fault_count = 0

    def __call__(self, **kwargs) -> dict:
        self.call_count += 1

        if random.random() < self.fault_rate:
            self.fault_count += 1
            print(f"    [FaultInjector] Injecting {self.fault_type} fault (call {self.call_count})")
            return {"status": "error", "error_type": self.fault_type,
                    "error": f"Injected {self.fault_type} for testing"}

        return self.tool_fn(**kwargs)

# Reliability test scenario
def run_reliability_test(agent_fn, tool, test_name: str, trials: int = 10):
    """Run multiple trials and measure success rate."""
    results = {"success": 0, "failed": 0, "escalated": 0}

    for i in range(trials):
        outcome = agent_fn(tool)
        status = outcome.get("status", "unknown")
        results[status if status in results else "failed"] += 1

    success_rate = results["success"] / trials * 100
    print(f"\nTest: {test_name}")
    print(f"  Trials: {trials}")
    print(f"  Success rate: {success_rate:.0f}%")
    print(f"  Results: {results}")
    return success_rate

# Mock tool and agent
def real_tool(query: str) -> dict:
    return {"status": "success", "result": f"Data for: {query}"}

faulty_tool = FaultInjector(real_tool, fault_rate=0.4, fault_type="timeout")

def simple_agent_with_retry(tool) -> dict:
    for attempt in range(3):
        result = tool(query="test query")
        if result["status"] == "success":
            return {"status": "success"}
    return {"status": "escalated"}

rate = run_reliability_test(simple_agent_with_retry, faulty_tool, "Agent with retry (40% fault rate)")
print(f"Pass: {'YES' if rate >= 90 else 'NO'} (target ≥ 90%)")