context-window-management

Strategies for managing LLM context windows through summarization, trimming, routing, and prioritization. Strategies for managing LLM context windows including summarization, trimming, routing, and avoiding context rot Different strategies based on context size When to use : Building any multi-turn conversation system interface ContextTier { maxTokens: number; strategy: 'full' | 'summarize' | 'rag'; model: string; } const TIERS: ContextTier[] = [ { maxTokens: 8000, strategy: 'full', model: 'claude-3-haiku' }, { maxTokens: 32000, strategy: 'full', model: 'claude-3-5-sonnet' }, { maxTokens: 100000, strategy: 'summarize', model: 'claude-3-5-sonnet' }, { maxTokens: Infinity, strategy: 'rag', model: 'claude-3-5-sonnet' } ]; async function selectStrategy(messages: Message[]): ContextTier { const tokens = await countTokens(messages); } async function prepareContext(messages: Message[]): PreparedContext { const tier = await selectStrategy(messages); } Place important content at start and end When to use : Constructing prompts with significant context // LLMs weight beginning and end more heavily // Structure prompts to leverage this function buildOptimalPrompt(components: { systemPrompt: string; criticalContext: string; conversationHistory: Message[]; currentQuery: string; }): string { // START: System instructions (always first) const parts = [components.systemPrompt]; } Summarize by...