Multi-Dimensional Assessment

Overview

Measuring only accuracy is like measuring car quality by top speed. What about safety, reliability, fuel efficiency, comfort?

Agent quality needs 8 dimensions.

-

The 8 Dimensions Framework

Dimension 1: Accuracy

class AccuracyMetric:
 """Does the agent produce correct outputs?"""

 def measure(self, agent, test_set):
 """
 Test set: (input, expected_output)
 """
 correct = 0

 for test_input, expected in test_set:
 output = agent.run(test_input)

 if self.matches(output, expected):
 correct += 1

 accuracy = correct / len(test_set)

 return {
 'accuracy': accuracy,
 'absolute_correct': correct,
 'absolute_total': len(test_set)
 }

 def matches(self, output, expected):
 """Flexible matching (fuzzy string, semantic, etc)"""
 return semantic_similarity(output, expected) > 0.9

Typical Range: 70-95% for production agents Target: Domain-dependent (code: 95%, customer service: 85%)

-

Dimension 2: Reliability

class ReliabilityMetric:
 """Is output consistent across runs?"""

 def measure(self, agent, test_set, runs=5):
 """Run each test multiple times"""

 consistency_scores = []

 for test_input, _ in test_set:
 outputs = []

 # Run same input N times
 for _ in range(runs):
 output = agent.run(test_input)
 outputs.append(output)

 # Measure consistency
 consistency = self.measure_consistency(outputs)
 consistency_scores.append(consistency)

 return {
 'reliability': sum(consistency_scores) / len(consistency_scores),
 'consistency_details': consistency_scores
 }

 def measure_consistency(self, outputs):
 """All outputs identical? Mostly similar? Random?"""
 if all(o == outputs[0] for o in outputs):
 return 1.0 # Perfect consistency

 # Semantic similarity
 similarities = []
 for i, o1 in enumerate(outputs):
 for o2 in outputs[i+1:]:
 similarities.append(semantic_similarity(o1, o2))

 return sum(similarities) / len(similarities)

Why It Matters: Unreliable agents are worse than wrong agents (you can't trust them) Typical Range: 75-98%

-

Dimension 3: Latency

class LatencyMetric:
 """How fast does the agent respond?"""

 def measure(self, agent, test_set):
 """Measure response time"""

 latencies = []

 for test_input, _ in test_set:
 start = time.time()
 agent.run(test_input)
 latency = time.time() - start
 latencies.append(latency)

 return {
 'p50': percentile(latencies, 50), # Median
 'p95': percentile(latencies, 95), # 95th percentile
 'p99': percentile(latencies, 99), # 99th percentile
 'max': max(latencies),
 'mean': sum(latencies) / len(latencies)
 }

Why It Matters: Users abandon slow agents Typical Range:

• Interactive: < 1 second
• Batch: < 30 seconds
• Async: < 1 minute

-

Dimension 4: Cost

class CostMetric:
 """What's the total cost per request?"""

 def measure(self, agent, test_set):
 """Track API costs"""

 costs = {
 'llm_calls': 0,
 'api_calls': 0,
 'compute': 0,
 'total': 0
 }

 for test_input, _ in test_set:
 call_cost = agent.run_and_track_cost(test_input)
 costs['total'] += call_cost.total
 costs['llm_calls'] += call_cost.llm
 costs['api_calls'] += call_cost.apis
 costs['compute'] += call_cost.compute

 avg_cost = costs['total'] / len(test_set)

 return {
 'cost_per_call': avg_cost,
 'cost_breakdown': costs,
 'annual_cost_1m_calls': avg_cost * 1_000_000
 }

Why It Matters: Cheap but broken is worse than expensive but reliable Typical Range: $0.001 - $0.10 per call

-

Dimension 5: Safety

class SafetyMetric:
 """Does agent respect policies and constraints?"""

 def measure(self, agent, policies, test_set):
 """Test policy compliance"""

 violations = {
 'data_access': 0,
 'cost_overrun': 0,
 'policy_violation': 0,
 'harmful_output': 0
 }

 for test_input, policy_constraint in test_set:
 try:
 output = agent.run(test_input)

 if not self.passes_policies(output, policies):
 violations['policy_violation'] += 1

 if self.contains_harmful_content(output):
 violations['harmful_output'] += 1

 except CostLimitExceeded:
 violations['cost_overrun'] += 1
 except UnauthorizedAccess:
 violations['data_access'] += 1

 compliance_rate = 1 - (sum(violations.values()) / len(test_set))

 return {
 'compliance_rate': compliance_rate,
 'violations': violations
 }

Why It Matters: One safety violation can lose customer trust Target: > 99.5% compliance

-

Dimension 6: Robustness

class RobustnessMetric:
 """How well does agent handle adversarial inputs?"""

 def measure(self, agent, test_set):
 """Test on modified/adversarial inputs"""

 results = {
 'typos': 0, # Misspelled words
 'injection': 0, # Prompt injection
 'outliers': 0, # Out-of-distribution
 'adversarial': 0 # Intentional attacks
 }

 for test_input, expected in test_set:
 # Test variant 1: with typos
 typo_input = self.add_typos(test_input)
 if agent.run(typo_input) still_correct:
 results['typos'] += 1

 # Test variant 2: with injection
 injection_input = self.add_injection(test_input)
 if agent.resists_injection(injection_input):
 results['injection'] += 1

 # Test variant 3: adversarial
 adversarial_input = self.make_adversarial(test_input)
 if agent.run(adversarial_input) robust:
 results['adversarial'] += 1

 robustness = sum(results.values()) / (len(results) * len(test_set))

 return {
 'robustness_score': robustness,
 'breakdown': results
 }

Why It Matters: Real users make typos, try injections, use adversarial inputs Typical Range: 60-85%

-

Dimension 7: Explainability

class ExplainabilityMetric:
 """Can we understand why the agent did something?"""

 def measure(self, agent, test_set):
 """Check explanation quality"""

 scores = []

 for test_input, _ in test_set:
 output, reasoning = agent.run_with_reasoning(test_input)

 # Score explanation quality
 score = self.rate_explanation(
 reasoning,
 output,
 test_input
)
 scores.append(score)

 explainability = sum(scores) / len(scores)

 return {
 'explainability': explainability,
 'reasoning_length': average_reasoning_length(agent),
 'token_efficiency': output_tokens / reasoning_tokens
 }

 def rate_explanation(self, reasoning, output, input_text):
 """Rate explanation on clarity, completeness, accuracy"""

 scores = {
 'addresses_input': self.covers_input(reasoning, input_text),
 'justifies_output': self.explains_output(reasoning, output),
 'step_by_step': self.is_sequential(reasoning),
 'clarity': self.measure_clarity(reasoning)
 }

 return sum(scores.values()) / len(scores)

Why It Matters: Debugging failures requires understanding decisions Typical Range: 60-90%

-

Dimension 8: Correctness

class CorrectnessMetric:
 """Is the output actually correct (vs just plausible)?"""

 def measure(self, agent, test_set, ground_truth_source):
 """Verify against authoritative source"""

 correct = 0
 partially_correct = 0
 wrong = 0

 for test_input, expected_output in test_set:
 output = agent.run(test_input)

 # Verify against ground truth
 verdict = ground_truth_source.verify(output, expected_output)

 if verdict.fully_correct:
 correct += 1
 elif verdict.partially_correct:
 partially_correct += 1
 else:
 wrong += 1

 return {
 'fully_correct': correct / len(test_set),
 'partially_correct': partially_correct / len(test_set),
 'wrong': wrong / len(test_set)
 }

Why It Matters: Accuracy might measure format, correctness verifies truth Typical Range: 70-95%

-

Aggregating 8 Dimensions

class ComprehensiveScore:
 def __init__(self, weights=None):
 # Customize importance of each dimension
 self.weights = weights or {
 'accuracy': 0.25,
 'reliability': 0.15,
 'latency': 0.10,
 'cost': 0.10,
 'safety': 0.15,
 'robustness': 0.10,
 'explainability': 0.10,
 'correctness': 0.05
 }

 def score(self, metrics):
 """Weighted aggregate of 8 dimensions"""

 score = 0
 for dimension, weight in self.weights.items():
 normalized = metrics[dimension] / 100 # 0-1 range
 score += normalized * weight

 return score * 100 # Back to 0-100

-

3 Warnings

Warning 1: Ignoring Tradeoffs

# WRONG
agent_a = {
 'accuracy': 95,
 'latency': 5s,
 'cost': $0.10
}

agent_b = {
 'accuracy': 92,
 'latency': 0.5s,
 'cost': $0.01
}

# Which is better? Need to weight tradeoffs

# RIGHT
weights = {
 'accuracy': 0.5, # Most important
 'latency': 0.3, # Some importance
 'cost': 0.2 # Less important
}

score_a = weighted_score(agent_a, weights)
score_b = weighted_score(agent_b, weights)
# Now can compare fairly

Warning 2: Measuring Wrong Proxy

# WRONG
# Measure "tokens used" as efficiency proxy
efficiency = output_tokens / input_tokens

# But doesn't measure what matters:
# - Latency? - Cost? - Accuracy?

# RIGHT
# Measure what actually matters
efficiency = {
 'cost_per_correct_answer': cost / accuracy,
 'latency_for_correct': latency_when_accurate,
 'tokens_per_value': tokens / value_delivered
}

Warning 3: Static Weights

# WRONG
weights = {
 'accuracy': 0.5,
 'cost': 0.1
} # Set once, never revisited

# But importance changes by domain:
# - Medical
# - Customer service

# RIGHT
weights = get_domain_specific_weights(domain)

# Medical
# CS

-

Last Updated: August 9, 2026