Failure Recovery

# Comprehensive failure recovery manager
class FailureRecoveryManager:
    def __init__(self, state: dict):
        self.state = state

    def recover(self, failed_step: dict, error: dict) -> dict:
        """
        Decide the recovery strategy for a failed step.
        Returns an action: retry | fallback | skip | escalate | compensate
        """
        error_type   = error.get("error_type", "unknown")
        retry_count  = self.state.get("retry_counts", {}).get(failed_step["id"], 0)
        is_optional  = failed_step.get("optional", False)
        has_fallback = bool(failed_step.get("fallback_action"))
        is_write     = failed_step.get("is_write_action", False)

        print(f"  Recovery for step '{failed_step['id']}' — error: {error_type}, retry: {retry_count}")

        # 1. Non-retryable errors → don't retry
        if error_type in ("auth", "permission", "validation"):
            if is_optional:
                return {"action": "skip", "reason": f"Non-retryable error on optional step"}
            return {"action": "escalate", "reason": f"Non-retryable {error_type} error"}

        # 2. Retryable error + retries remaining → retry
        if error_type in ("timeout", "network", "server") and retry_count < 3:
            new_count = retry_count + 1
            self.state.setdefault("retry_counts", {})[failed_step["id"]] = new_count
            return {"action": "retry", "wait_seconds": 2 ** retry_count,
                    "reason": f"Retryable {error_type} (attempt {new_count}/3)"}

        # 3. Retries exhausted + fallback available → use fallback
        if has_fallback:
            return {"action": "fallback",
                    "fallback_action": failed_step["fallback_action"],
                    "reason": "Retries exhausted — using fallback"}

        # 4. Step is optional → skip it
        if is_optional:
            return {"action": "skip", "reason": "Non-critical step failed — skipping"}

        # 5. Write action that partially completed → compensate
        if is_write and self.state.get(f"{failed_step['id']}_partial"):
            return {"action": "compensate",
                    "compensation_step": failed_step.get("compensation_action"),
                    "reason": "Write partially completed — compensating"}

        # 6. Nothing worked → escalate
        return {"action": "escalate", "reason": "All recovery options exhausted"}

state = {"retry_counts": {}}
manager = FailureRecoveryManager(state)

step = {"id": "S3", "optional": False, "fallback_action": "use_cache", "is_write_action": False}
error = {"error_type": "timeout"}

for i in range(5):
    decision = manager.recover(step, error)
    print(f"    Decision: {decision['action']} — {decision['reason']}")
    if decision["action"] != "retry":
        break