"Context Window & Token Limits Explained: A Developer's Guide to Every AI Model"

# Count tokens with tiktoken (OpenAI models)
import tiktoken

encoder = tiktoken.encoding_for_model("gpt-4o")
text = "Hello, how many tokens is this sentence?"
tokens = encoder.encode(text)
print(f"Token count: {len(tokens)}")  # Output: 8

┌────────────────────────────────────────────────────┐
│                    Context Window                    │
│                                                      │
│  ┌────────────┐                    ┌────────────┐   │
│  │  Beginning  │  ← Model pays     │    End     │   │
│  │  (Strong)   │    attention here  │  (Strong)  │   │
│  └────────────┘                    └────────────┘   │
│                                                      │
│              ┌──────────────────┐                    │
│              │     Middle       │ ← Information      │
│              │   (Weaker)      │   often missed     │
│              └──────────────────┘                    │
│                                                      │
└────────────────────────────────────────────────────┘

def prune_conversation(messages, max_tokens=100000):
    """Keep conversation within token budget."""
    # Always keep system message
    system = [m for m in messages if m["role"] == "system"]
    conversation = [m for m in messages if m["role"] != "system"]
    
    # Count tokens (simplified)
    total_tokens = sum(len(m["content"]) // 4 for m in messages)
    
    # Remove oldest messages until within budget
    while total_tokens > max_tokens and len(conversation) > 2:
        removed = conversation.pop(0)
        total_tokens -= len(removed["content"]) // 4
    
    return system + conversation

from openai import OpenAI

client = OpenAI(
    api_key="YOUR_CRAZYROUTER_KEY",
    base_url="https://crazyrouter.com/v1"
)

def summarize_history(messages):
    """Compress conversation history into a summary."""
    history_text = "\n".join(
        f"{m['role']}: {m['content']}" for m in messages
    )
    
    response = client.chat.completions.create(
        model="gpt-4o-mini",  # Cheap model for summarization
        messages=[
            {"role": "system", "content": "Summarize this conversation concisely, preserving key decisions, facts, and context."},
            {"role": "user", "content": history_text}
        ],
        max_tokens=500
    )
    
    return {
        "role": "system",
        "content": f"Previous conversation summary: {response.choices[0].message.content}"
    }

def chunk_document(text, chunk_size=4000, overlap=200):
    """Split a document into overlapping chunks for processing."""
    words = text.split()
    chunks = []
    
    for i in range(0, len(words), chunk_size - overlap):
        chunk = " ".join(words[i:i + chunk_size])
        chunks.append(chunk)
    
    return chunks

def process_long_document(document, question):
    """Process a document that exceeds context window."""
    chunks = chunk_document(document)
    
    # First pass: extract relevant information from each chunk
    relevant_parts = []
    for i, chunk in enumerate(chunks):
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": "Extract any information relevant to the question. Return 'NOT_RELEVANT' if nothing is relevant."},
                {"role": "user", "content": f"Question: {question}\n\nText chunk {i+1}:\n{chunk}"}
            ],
            max_tokens=500
        )
        result = response.choices[0].message.content
        if "NOT_RELEVANT" not in result:
            relevant_parts.append(result)
    
    # Second pass: synthesize the answer
    synthesis = client.chat.completions.create(
        model="gpt-4o",  # Use a stronger model for synthesis
        messages=[
            {"role": "system", "content": "Synthesize a comprehensive answer from these extracted passages."},
            {"role": "user", "content": f"Question: {question}\n\nRelevant passages:\n" + "\n---\n".join(relevant_parts)}
        ]
    )
    
    return synthesis.choices[0].message.content

# Simplified RAG pattern
from openai import OpenAI

client = OpenAI(
    api_key="YOUR_CRAZYROUTER_KEY",
    base_url="https://crazyrouter.com/v1"
)

def embed_text(text):
    """Generate embeddings for semantic search."""
    response = client.embeddings.create(
        model="text-embedding-3-small",
        input=text
    )
    return response.data[0].embedding

def rag_query(question, knowledge_base):
    """Answer using only the most relevant context."""
    # 1. Embed the question
    q_embedding = embed_text(question)
    
    # 2. Find top-k similar documents (cosine similarity)
    relevant_docs = find_similar(q_embedding, knowledge_base, top_k=5)
    
    # 3. Build focused context (much smaller than full knowledge base)
    context = "\n\n".join(doc["text"] for doc in relevant_docs)
    
    # 4. Generate answer with focused context
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "Answer based on the provided context. Cite sources when possible."},
            {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {question}"}
        ]
    )
    
    return response.choices[0].message.content

# Quick token estimation across providers
def estimate_tokens(text, language="en"):
    if language == "en":
        return len(text.split()) * 1.3  # ~1.3 tokens per word
    elif language in ["zh", "ja", "ko"]:
        return len(text) * 0.7  # ~0.7 tokens per character
    else:
        return len(text) / 4  # General estimate