Error Recovery: Building Resilient Systems¶
Overview¶
Production systems fail. The question isn't "if" but "when" and "how quickly can you recover?"
Error Classification¶
Error Categories¶
class ErrorClassification:
"""Categorize errors by recovery approach"""
def classify(self, error) -> str:
"""Determine error type"""
if isinstance(error, TransientError):
# Temporary failure (network blip, rate limit)
return 'transient'
elif isinstance(error, PermanentError):
# Won't succeed with retry (invalid input, not found)
return 'permanent'
elif isinstance(error, TimeoutError):
# Took too long
return 'timeout'
elif isinstance(error, ResourceError):
# Out of resources (OOM, quota exceeded)
return 'resource'
elif isinstance(error, SystemError):
# Internal system error (bug in code)
return 'system'
else:
return 'unknown'
def recovery_strategy(self, error_type):
"""Different strategy for each error type"""
strategies = {
'transient': 'retry_with_backoff',
'permanent': 'fallback_or_escalate',
'timeout': 'circuit_break',
'resource': 'queue_or_reject',
'system': 'escalate_to_humans'
}
return strategies[error_type]
Recovery Strategies¶
Strategy 1: Retry with Exponential Backoff¶
class RetryWithBackoff:
"""Retry transient errors with increasing delays"""
def execute_with_retry(self, func, max_retries=3):
"""Retry with exponential backoff"""
for attempt in range(max_retries):
try:
return func()
except TransientError as e:
if attempt < max_retries - 1:
# Exponential backoff: 1s, 2s, 4s, 8s...
wait_time = min(2 ** attempt, 60)
# Add jitter to prevent thundering herd
jitter = random.uniform(0, 0.1 * wait_time)
time.sleep(wait_time + jitter)
else:
raise
Strategy 2: Fallback¶
class FallbackRecovery:
"""Use alternative when primary fails"""
def execute_with_fallback(self, request):
"""Try primary, fallback on failure"""
try:
# Try expensive/complex path
return self.expensive_solution(request)
except Exception as e:
# Fall back to cheaper/simpler solution
return self.cheap_solution(request)
def expensive_solution(self, request):
"""Complex reasoning, expensive"""
return self.complex_agent.solve(request)
def cheap_solution(self, request):
"""Simple heuristic, cheap"""
return self.heuristic_solver.solve(request)
Strategy 3: Circuit Breaker¶
class CircuitBreakerRecovery:
"""Stop calling failing service"""
def __init__(self, failure_threshold=5, timeout=60):
self.state = 'closed' # Normal
self.failure_count = 0
self.last_failure = None
self.threshold = failure_threshold
self.timeout = timeout
def call(self, func):
"""Call with circuit breaker protection"""
if self.state == 'open':
# Circuit is open, don't call
if time.time() - self.last_failure > self.timeout:
# Try again (half-open state)
self.state = 'half-open'
else:
raise CircuitBreakerOpen("Service unavailable")
try:
result = func()
# Success - close circuit
self.state = 'closed'
self.failure_count = 0
return result
except Exception as e:
# Failure - increment counter
self.failure_count += 1
self.last_failure = time.time()
if self.failure_count >= self.threshold:
self.state = 'open' # Open circuit
raise
Graceful Degradation¶
Degradation Strategies¶
class GracefulDegradation:
"""Reduce quality rather than fail completely"""
def handle_request(self, request):
"""Try progressively simpler strategies"""
try:
# Attempt 1: Full capability
return self.full_solution(request)
except Exception:
try:
# Attempt 2: Simplified solution
return self.simplified_solution(request)
except Exception:
try:
# Attempt 3: Cached result
return self.cached_result(request)
except Exception:
# Attempt 4: Default response
return self.default_response(request)
def full_solution(self, request):
"""Best quality, most expensive"""
return self.complex_agent.solve(request)
def simplified_solution(self, request):
"""Medium quality, cheaper"""
return self.fast_agent.solve(request)
def cached_result(self, request):
"""Old result, free"""
return self.cache.get(request.id)
def default_response(self, request):
"""Basic response, always works"""
return {'status': 'queued', 'estimate': '1 hour'}
Health Checks & Healing¶
Proactive Health Monitoring¶
class HealthCheckHealing:
"""Monitor health, heal before failure"""
def run_health_check(self):
"""Periodically check system health"""
health = {
'llm_available': self.check_llm(),
'tools_available': self.check_tools(),
'memory_available': self.check_memory(),
'disk_available': self.check_disk(),
'database_responsive': self.check_database()
}
# Identify unhealthy components
unhealthy = [k for k, v in health.items() if not v]
if unhealthy:
# Heal proactively
self.heal(unhealthy)
# Alert ops
self.notify_ops(unhealthy)
return health
def heal(self, components):
"""Auto-healing when possible"""
for component in components:
if component == 'memory_available':
# Clear caches
self.clear_caches()
elif component == 'database_responsive':
# Restart database connection
self.restart_database_connection()
elif component == 'tools_available':
# Restart tools service
self.restart_tools_service()
3 Warnings ⚠️¶
Warning 1: Infinite Retry Loops¶
# ❌ WRONG
while True:
try:
result = call_api()
break
except:
continue # Retry forever!
# If service is down, loops infinitely
# Consumes resources
# ✅ RIGHT
max_retries = 3
for attempt in range(max_retries):
try:
result = call_api()
break
except TransientError:
if attempt < max_retries - 1:
time.sleep(2 ** attempt)
else:
raise
# Bounded retries
Warning 2: Hiding Real Errors¶
# ❌ WRONG
try:
critical_operation()
except:
pass # Silently ignore!
# Error disappears
# Data corruption possible
# ✅ RIGHT
try:
critical_operation()
except PermanentError:
raise # Don't hide permanent errors
except TransientError:
retry_later() # Only retry on transient
Warning 3: Cascading Failures¶
# ❌ WRONG
# No circuit breaker
for request in requests:
result = call_failing_service()
# Service down, all requests fail
# Overwhelms service with retries
# ✅ RIGHT
# Circuit breaker prevents cascade
breaker = CircuitBreaker()
for request in requests:
try:
result = breaker.call(call_service)
except CircuitBreakerOpen:
# Service down, skip gracefully
queue_for_later(request)
# Protects service from overload
Last Updated: August 9, 2026