Testing Tool Selection

# Tool selection tests using evaluation scoring
class ToolSelectionEvaluator:
    """Evaluate whether the agent selected the best tool for each scenario."""

    TEST_CASES = [
        {
            "scenario": "User asks for current weather in London",
            "expected_tool": "get_weather",
            "acceptable_tools": {"get_weather", "web_search"},
            "unacceptable_tools": {"send_email", "delete_record", "issue_refund"},
        },
        {
            "scenario": "User asks to refund order ORD-123",
            "expected_tool": "issue_refund",
            "acceptable_tools": {"issue_refund", "get_order"},
            "unacceptable_tools": {"web_search", "send_email"},
        },
        {
            "scenario": "User asks what time it is in Tokyo",
            "expected_tool": "get_time",
            "acceptable_tools": {"get_time", "web_search"},
            "unacceptable_tools": {"issue_refund", "delete_record"},
        },
    ]

    @classmethod
    def evaluate(cls, agent_tool_selector) -> dict:
        """
        Run all test cases against an agent tool selector.
        Returns pass/fail counts and details.
        """
        results = {"passed": 0, "failed": 0, "cases": []}

        for case in cls.TEST_CASES:
            # Call the agent selector (mocked here)
            selected_tool = agent_tool_selector(case["scenario"])

            is_acceptable  = selected_tool in case["acceptable_tools"]
            is_unacceptable = selected_tool in case["unacceptable_tools"]
            passed = is_acceptable and not is_unacceptable

            results["cases"].append({
                "scenario":     case["scenario"][:40],
                "expected":     case["expected_tool"],
                "selected":     selected_tool,
                "passed":       passed,
            })

            if passed:
                results["passed"] += 1
                print(f"  ✓ {case['scenario'][:40]} → {selected_tool}")
            else:
                results["failed"] += 1
                print(f"  ✗ {case['scenario'][:40]} → {selected_tool} (expected: {case['expected_tool']})")

        results["pass_rate"] = results["passed"] / len(cls.TEST_CASES) * 100
        return results

# Mock agent that gets the first two right but fails the third
def mock_agent_selector(scenario: str) -> str:
    if "weather" in scenario:   return "get_weather"
    if "refund" in scenario:    return "issue_refund"
    if "time" in scenario:      return "web_search"  # wrong but acceptable
    return "web_search"

results = ToolSelectionEvaluator.evaluate(mock_agent_selector)
print(f"\nPass rate: {results['pass_rate']:.0f}% ({results['passed']}/{len(ToolSelectionEvaluator.TEST_CASES)})")