Custom Evaluation Frameworks¶
Overview¶
Generic benchmarks don't capture what matters for YOUR use case. Custom evaluation frameworks measure what actually counts.
Building Domain-Specific Evaluation¶
Step 1: Define Success Criteria¶
class DomainEvaluationFramework:
"""Template for custom evaluation"""
def __init__(self, domain: str):
self.domain = domain
# Define success for THIS domain
self.success_criteria = self.define_success()
def define_success(self) -> dict:
"""Domain-specific definition of 'correct'"""
if self.domain == 'customer_support':
return {
'customer_satisfied': True,
'issue_resolved': True,
'response_time': '< 2 minutes',
'tone_appropriate': True,
'no_escalation_needed': True
}
elif self.domain == 'code_generation':
return {
'code_runs': True,
'passes_tests': True,
'follows_style': True,
'well_documented': True,
'no_security_issues': True
}
elif self.domain == 'medical_diagnosis':
return {
'diagnosis_accurate': True,
'reasoning_sound': True,
'no_harmful_advice': True,
'appropriate_escalation': True,
'follows_guidelines': True
}
Step 2: Construct Test Sets¶
class TestSetConstruction:
"""Build evaluation data"""
def build_comprehensive_set(self, domain: str):
"""Balanced mix of cases"""
test_cases = {
# Easy cases (should always work)
'easy': self.generate_easy_cases(domain, count=100),
# Medium cases (require some reasoning)
'medium': self.generate_medium_cases(domain, count=100),
# Hard cases (rare, important)
'hard': self.generate_hard_cases(domain, count=50),
# Edge cases (corner scenarios)
'edge': self.generate_edge_cases(domain, count=50),
# Adversarial (users trying to break it)
'adversarial': self.generate_adversarial(domain, count=50)
}
return test_cases
def generate_hard_cases(self, domain: str, count: int):
"""Cases with multiple correct paths"""
if domain == 'customer_support':
return [
{
'input': 'Billing issue with conflicting charges',
'correct_paths': [
'Refund + apology',
'Credit for future use',
'Partial refund + discount'
],
'wrong_paths': [
'Deny issue exists',
'Blame customer',
'Demand payment'
]
}
]
def generate_edge_cases(self, domain: str, count: int):
"""Rare but important scenarios"""
if domain == 'code_generation':
return [
{
'input': 'Generate thread-safe counter',
'requirements': ['Thread-safe', 'Lock-free if possible'],
'common_mistakes': [
'Race condition',
'Deadlock',
'Performance issue'
]
}
]
-
Step 3: Evaluation Harness¶
class EvaluationHarness:
"""Run tests and collect metrics"""
def evaluate(self, agent, test_set, domain: str):
"""Comprehensive evaluation"""
results = {
'by_difficulty': {},
'by_category': {},
'failures': [],
'metrics': {}
}
for difficulty, test_cases in test_set.items():
difficulty_results = []
for test_case in test_cases:
try:
# Run the test
output = agent.run(test_case['input'])
# Evaluate output
score = self.grade_output(
output,
test_case,
domain
)
difficulty_results.append(score)
except Exception as e:
results['failures'].append({
'test': test_case['input'],
'error': str(e),
'difficulty': difficulty
})
results['by_difficulty'][difficulty] = {
'accuracy': sum(difficulty_results) / len(difficulty_results),
'count': len(difficulty_results)
}
return results
def grade_output(self, output, test_case, domain: str) -> float:
"""Domain-specific grading"""
if domain == 'customer_support':
score = 0
# Check resolution quality
if self.resolves_issue(output, test_case):
score += 0.4
# Check tone
if self.appropriate_tone(output):
score += 0.3
# Check no escalation needed
if not self.requires_escalation(output):
score += 0.3
return score
elif domain == 'code_generation':
score = 0
# Test runs
if self.code_runs(output):
score += 0.3
# Tests pass
if self.passes_tests(output, test_case):
score += 0.4
# No security issues
if not self.has_security_issues(output):
score += 0.3
return score
-
Step 4: Analyze Failure Modes¶
class FailureModeAnalysis:
"""Understand why agent fails"""
def analyze(self, agent, test_set, results):
"""Categorize failures"""
failure_modes = {
'reasoning_error': 0,
'tool_misuse': 0,
'safety_violation': 0,
'incomplete_solution': 0,
'hallucination': 0,
'timeout': 0
}
for failure in results['failures']:
test_case = self.find_test_case(failure['test'], test_set)
# Categorize failure
category = self.classify_failure(
failure,
test_case,
agent
)
failure_modes[category] += 1
# Print analysis
for mode, count in failure_modes.items():
percentage = (count / len(results['failures'])) * 100
print(f"{mode}: {count} ({percentage:.1f}%)")
# Identify top failure mode
top_mode = max(failure_modes, key=failure_modes.get)
print(f"\nTop failure: {top_mode}")
print("Action: Fix this and retest")
3 Warnings¶
Warning 1: Evaluation Bias¶
# WRONG
# Build test set with only "happy path"
test_set = [
'What is 2+2?', # Easy
'Sum 1-100?', # Medium
'Optimize algorithm?' # Hard
]
# Accuracy
# RIGHT
# Balanced test set
test_set = {
'easy': [...], # 30%
'medium': [...], # 40%
'hard': [...], # 20%
'edge_cases': [...], # 5%
'adversarial': [...] # 5%
}
# More realistic accuracy
Warning 2: Overfitting to Custom Eval¶
# WRONG
# Agent is optimized for your specific test set
agent = optimize_for_eval(agent, your_test_set)
accuracy_on_your_eval = 95%
accuracy_on_user_data = 42% # Oops!
# RIGHT
# Keep eval set separate
eval_set_v1 = build_eval_set() # Use for development
eval_set_v2 = build_eval_set() # Hold out for testing
agent = train_on_v1(agent)
final_accuracy = evaluate_on_v2(agent)
Warning 3: Expensive Evaluation¶
# WRONG
class SlowEvaluation:
def evaluate(self, agent):
# Run agent 10,000 times
for i in range(10000):
agent.run(test_case)
# Takes 5 hours!
# Runs only once per release
# RIGHT
class FastEvaluation:
def evaluate(self, agent):
# Stratified sample: 500 cases
for test_case in stratified_sample(500):
agent.run(test_case)
# Takes 10 minutes!
# Can run on every commit
-
Last Updated: August 9, 2026