Context Management: Controlling Token Costs¶
Overview¶
LLM context windows are expensive. A single call with 100K tokens costs 10x more than one with 10K tokens.
Smart context management is the difference between profitable and bankrupt.
Context Window Economics¶
The Math¶
Context Size | Cost per Call | 1M Calls
- ┼───────────────┼──────────
4K tokens | $0.01 | $10k
16K tokens | $0.04 | $40k
100K tokens | $0.25 | $250k ⚠️
200K tokens | $0.50 | $500k ⚠️⚠️
Key Insight: Context size drives cost exponentially
Compression Strategies¶
Strategy 1: Summarization¶
class SummarizationCompression:
"""Summarize old context to reduce size"""
def compress_history(self, conversation_history):
"""Compress old messages"""
old_messages = conversation_history[:-10] # Keep last 10
recent_messages = conversation_history[-10:]
if len(old_messages) > 20:
# Old messages are bulky
summary = self.summarize(old_messages)
return [
{'role': 'system', 'content': f"Previous context: {summary}"},
*recent_messages
]
return conversation_history
def summarize(self, messages) -> str:
"""Create compressed summary"""
# Use efficient summarization model
summary = self.cheap_model.summarize(
f"Summarize this conversation:\n{messages}"
)
return summary
Token Savings: 80% reduction in old context
Cost: Small (cheap model for summarization)
Strategy 2: Retrieval-Augmented Generation (RAG)¶
class RAGContextManagement:
"""Only include relevant context"""
def __init__(self):
self.vector_db = VectorDB()
def select_context(self, query):
"""Retrieve only relevant messages"""
# Find most relevant past messages
relevant = self.vector_db.search(
query=query,
limit=5, # Only top 5
similarity_threshold=0.7
)
return relevant
def build_prompt_with_context(self, query, task):
"""Build minimal prompt with necessary context"""
context = self.select_context(query)
prompt = f"""
Current task: {task}
Relevant context from history:
{format_context(context)}
User query: {query}
"""
return prompt
Token Savings: 70-90% reduction
Cost: Vector DB operations (small)
Strategy 3: Windowed Context¶
class WindowedContextManagement:
"""Keep sliding window of recent context"""
def __init__(self, window_size=20):
self.window = []
self.window_size = window_size
def add_message(self, message):
"""Add to window, drop old messages"""
self.window.append(message)
# Keep only recent N messages
if len(self.window) > self.window_size:
self.window = self.window[-self.window_size:]
def get_context(self):
"""Return current window"""
return self.window
Token Savings: Fixed window = predictable cost
Tradeoff: Lose old context after N messages
Token Budgeting¶
Budget Allocation¶
class TokenBudgetManager:
"""Allocate tokens across components"""
def __init__(self, total_budget=100000):
self.total_budget = total_budget
self.allocation = {
'system_prompt': 2000, # Fixed
'context': 50000, # Variable
'query': 5000, # User input
'reasoning': 30000, # Agent thinking
'response': 13000 # Output
}
def validate_request(self, request):
"""Check if request fits budget"""
estimated_tokens = self.estimate_tokens(request)
if estimated_tokens > self.total_budget:
# Too large! Need to compress
return False, estimated_tokens
return True, estimated_tokens
def estimate_tokens(self, request):
"""Estimate token usage"""
tokens = 0
tokens += len(self.allocation['system_prompt'])
tokens += self.estimate_context_tokens(request)
tokens += len(request.query.split()) * 1.3 # Approx
tokens += self.allocation['reasoning']
return tokens
Smart Truncation¶
Intelligent Truncation¶
class IntelligentTruncation:
"""Smart truncation that preserves important info"""
def truncate_context(self, messages, max_tokens):
"""Keep most important messages"""
# Score each message by importance
scored = []
for msg in messages:
score = self.score_importance(msg)
tokens = self.estimate_tokens(msg)
scored.append((msg, score, tokens))
# Sort by importance
scored.sort(key=lambda x: x[1], reverse=True)
# Keep highest importance until budget exceeded
selected = []
total_tokens = 0
for msg, score, tokens in scored:
if total_tokens + tokens <= max_tokens:
selected.append(msg)
total_tokens += tokens
# Re-order by original sequence
return sorted(selected, key=lambda x: messages.index(x))
def score_importance(self, message) -> float:
"""Rate importance of message"""
score = 0
if message['role'] == 'user':
score += 0.5 # User messages matter
if 'question' in message['content'].lower():
score += 0.3 # Questions matter
if 'decision' in message['content'].lower():
score += 0.3 # Decisions matter
if 'error' in message['content'].lower():
score += 0.2 # Errors worth tracking
return score
3 Warnings ⚠️¶
Warning 1: Losing Critical Context¶
# ❌ WRONG
# Truncate aggressively
context = get_context()
if len(context) > 10000:
context = context[-5000:] # Keep only last 5K
# Lost important constraint from earlier!
# Agent violates policy
# ✅ RIGHT
# Smart truncation that preserves constraints
context = select_critical_context(
all_context,
keywords=['must', 'cannot', 'policy', 'constraint']
)
context = compress_non_critical(context)
# Policies preserved, cost reduced
Warning 2: Unbounded Context Growth¶
# ❌ WRONG
# Add to context forever
while True:
message = get_message()
context.append(message)
# Context grows indefinitely!
# Cost: $0.01/call initially
# Cost: $1/call after 100 iterations ⚠️
# ✅ RIGHT
# Manage context size
context = manage_context(
context,
max_size=50000,
compression='summarization'
)
# Cost stays ~$0.05/call always
Warning 3: No Budget Monitoring¶
# ❌ WRONG
# Just call LLM without tracking
response = llm.call(prompt)
# No visibility into token usage
# Costs spiral unexpectedly
# ✅ RIGHT
# Monitor and alert
budget_manager = TokenBudgetManager(budget=10000)
estimated = budget_manager.estimate_tokens(prompt)
if estimated > budget_manager.remaining:
compress_context()
else:
response = llm.call(prompt)
budget_manager.deduct(response.tokens_used)
Last Updated: August 9, 2026