Context Management

Overview

LLM context windows are expensive. A single call with 100K tokens costs 10x more than one with 10K tokens.

Smart context management is the difference between profitable and bankrupt.

---

Context Window Economics

The Math

Context Size| Cost per Call| 1M Calls
- ┼───────────────┼──────────
4K tokens| $0.01| $10k
16K tokens| $0.04| $40k
100K tokens| $0.25| $250k 
200K tokens| $0.50| $500k 

Key Insight: Context size drives cost exponentially

---

Compression Strategies

Strategy 1: Summarization

class SummarizationCompression:
 """Summarize old context to reduce size"""

 def compress_history(self, conversation_history):
 """Compress old messages"""

 old_messages = conversation_history[:-10] # Keep last 10
 recent_messages = conversation_history[-10:]

 if len(old_messages) > 20:
 # Old messages are bulky
 summary = self.summarize(old_messages)

 return [
 {'role': 'system', 'content': f"Previous context: {summary}"},
 *recent_messages
]

 return conversation_history

 def summarize(self, messages) -> str:
 """Create compressed summary"""

 # Use efficient summarization model
 summary = self.cheap_model.summarize(
 f"Summarize this conversation:\n{messages}"
)

 return summary

Token Savings: 80% reduction in old context Cost: Small (cheap model for summarization)

---

Strategy 2: Retrieval-Augmented Generation (RAG)

class RAGContextManagement:
 """Only include relevant context"""

 def __init__(self):
 self.vector_db = VectorDB()

 def select_context(self, query):
 """Retrieve only relevant messages"""

 # Find most relevant past messages
 relevant = self.vector_db.search(
 query=query,
 limit=5, # Only top 5
 similarity_threshold=0.7
)

 return relevant

 def build_prompt_with_context(self, query, task):
 """Build minimal prompt with necessary context"""

 context = self.select_context(query)

 prompt = f"""
 Current task: {task}

 Relevant context from history:
 {format_context(context)}

 User query: {query}
 """

 return prompt

Token Savings: 70-90% reduction Cost: Vector DB operations (small)

---

Strategy 3: Windowed Context

class WindowedContextManagement:
 """Keep sliding window of recent context"""

 def __init__(self, window_size=20):
 self.window = []
 self.window_size = window_size

 def add_message(self, message):
 """Add to window, drop old messages"""

 self.window.append(message)

 # Keep only recent N messages
 if len(self.window) > self.window_size:
 self.window = self.window[-self.window_size:]

 def get_context(self):
 """Return current window"""
 return self.window

Token Savings: Fixed window = predictable cost Tradeoff: Lose old context after N messages

---

Token Budgeting

Budget Allocation

class TokenBudgetManager:
 """Allocate tokens across components"""

 def __init__(self, total_budget=100000):
 self.total_budget = total_budget
 self.allocation = {
 'system_prompt': 2000, # Fixed
 'context': 50000, # Variable
 'query': 5000, # User input
 'reasoning': 30000, # Agent thinking
 'response': 13000 # Output
 }

 def validate_request(self, request):
 """Check if request fits budget"""

 estimated_tokens = self.estimate_tokens(request)

 if estimated_tokens > self.total_budget:
 # Too large! Need to compress
 return False, estimated_tokens

 return True, estimated_tokens

 def estimate_tokens(self, request):
 """Estimate token usage"""

 tokens = 0
 tokens += len(self.allocation['system_prompt'])
 tokens += self.estimate_context_tokens(request)
 tokens += len(request.query.split()) * 1.3 # Approx
 tokens += self.allocation['reasoning']

 return tokens

---

Smart Truncation

Intelligent Truncation

class IntelligentTruncation:
 """Smart truncation that preserves important info"""

 def truncate_context(self, messages, max_tokens):
 """Keep most important messages"""

 # Score each message by importance
 scored = []

 for msg in messages:
 score = self.score_importance(msg)
 tokens = self.estimate_tokens(msg)
 scored.append((msg, score, tokens))

 # Sort by importance
 scored.sort(key=lambda x: x[1], reverse=True)

 # Keep highest importance until budget exceeded
 selected = []
 total_tokens = 0

 for msg, score, tokens in scored:
 if total_tokens + tokens <= max_tokens:
 selected.append(msg)
 total_tokens += tokens

 # Re-order by original sequence
 return sorted(selected, key=lambda x: messages.index(x))

 def score_importance(self, message) -> float:
 """Rate importance of message"""

 score = 0

 if message['role'] == 'user':
 score += 0.5 # User messages matter

 if 'question' in message['content'].lower():
 score += 0.3 # Questions matter

 if 'decision' in message['content'].lower():
 score += 0.3 # Decisions matter

 if 'error' in message['content'].lower():
 score += 0.2 # Errors worth tracking

 return score

-

3 Warnings

Warning 1: Losing Critical Context

# WRONG
# Truncate aggressively
context = get_context()
if len(context) > 10000:
 context = context[-5000:] # Keep only last 5K

# Lost important constraint from earlier!
# Agent violates policy

# RIGHT
# Smart truncation that preserves constraints
context = select_critical_context(
 all_context,
 keywords=['must', 'cannot', 'policy', 'constraint']
)
context = compress_non_critical(context)
# Policies preserved, cost reduced

Warning 2: Unbounded Context Growth

# WRONG
# Add to context forever
while True:
 message = get_message()
 context.append(message)
 # Context grows indefinitely!

# Cost
# Cost

# RIGHT
# Manage context size
context = manage_context(
 context,
 max_size=50000,
 compression='summarization'
)

# Cost stays ~$0.05/call always

Warning 3: No Budget Monitoring

# WRONG
# Just call LLM without tracking
response = llm.call(prompt)
# No visibility into token usage

# Costs spiral unexpectedly

# RIGHT
# Monitor and alert
budget_manager = TokenBudgetManager(budget=10000)
estimated = budget_manager.estimate_tokens(prompt)

if estimated > budget_manager.remaining:
 compress_context()
else:
 response = llm.call(prompt)
 budget_manager.deduct(response.tokens_used)

-

Last Updated: August 9, 2026