AI Inference Speed Benchmark 2026: Tokens Per Second Compared

High Speed, Lower Quality:
  Gemini 2.5 Flash Lite → Claude Haiku 4.5 → GPT-5 Mini
  
  Best for: autocomplete, simple Q&A, data extraction

Balanced Speed + Quality:
  Gemini 2.5 Flash → Claude Sonnet 4.5 → DeepSeek V3.2
  
  Best for: chatbots, content generation, analysis

High Quality, Slower:
  Claude Opus 4.6 → GPT-5.2 → Kimi K2 Thinking
  
  Best for: complex reasoning, code review, research

from openai import OpenAI

client = OpenAI(
    api_key="your-crazyrouter-key",
    base_url="https://crazyrouter.com/v1"
)

# Always stream — users see output immediately
stream = client.chat.completions.create(
    model="claude-sonnet-4-5",
    messages=[{"role": "user", "content": "Explain streaming APIs"}],
    stream=True  # Critical for perceived latency
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

def select_model(task_type: str, quality_requirement: str) -> str:
    """Route to the right model based on task needs."""
    
    if task_type == "autocomplete" or quality_requirement == "low":
        return "gemini-2.5-flash-lite"  # Fastest
    
    elif task_type == "chat" and quality_requirement == "medium":
        return "claude-haiku-4-5"  # Fast + capable
    
    elif task_type == "reasoning" or quality_requirement == "high":
        return "claude-opus-4-6"  # Best quality
    
    else:
        return "claude-sonnet-4-5"  # Good default

import asyncio
from openai import AsyncOpenAI

client = AsyncOpenAI(
    api_key="your-crazyrouter-key",
    base_url="https://crazyrouter.com/v1"
)

async def fast_completion_with_fallback(prompt: str, timeout: float = 2.0):
    """Try fast model first, fall back to reliable model on timeout."""
    
    try:
        # Try the fastest model with a strict timeout
        response = await asyncio.wait_for(
            client.chat.completions.create(
                model="gemini-2.5-flash-lite",
                messages=[{"role": "user", "content": prompt}],
                max_tokens=512
            ),
            timeout=timeout
        )
        return response.choices[0].message.content
    
    except asyncio.TimeoutError:
        # Fall back to more reliable model
        response = await client.chat.completions.create(
            model="claude-sonnet-4-5",
            messages=[{"role": "user", "content": prompt}],
            max_tokens=512
        )
        return response.choices[0].message.content

# Claude supports caching via cache_control
response = client.chat.completions.create(
    model="claude-opus-4-6",
    messages=[
        {
            "role": "system",
            "content": [
                {
                    "type": "text",
                    "text": "You are an expert code reviewer. " + large_codebase_context,
                    "cache_control": {"type": "ephemeral"}  # Cache this
                }
            ]
        },
        {"role": "user", "content": "Review this new function..."}
    ]
)

import asyncio

async def parallel_analysis(code_files: list[str]):
    """Analyze multiple files in parallel instead of sequentially."""
    
    tasks = [
        client.chat.completions.create(
            model="claude-haiku-4-5",
            messages=[
                {"role": "user", "content": f"Review this code:\n{file}"}
            ]
        )
        for file in code_files
    ]
    
    # All requests fire simultaneously
    responses = await asyncio.gather(*tasks)
    return [r.choices[0].message.content for r in responses]

import OpenAI from 'openai';

const client = new OpenAI({
  apiKey: process.env.CRAZYROUTER_API_KEY,
  baseURL: 'https://crazyrouter.com/v1',
});

async function measureLatency(model, prompt) {
  const start = Date.now();
  let firstTokenTime = null;
  let tokenCount = 0;
  
  const stream = await client.chat.completions.create({
    model,
    messages: [{ role: 'user', content: prompt }],
    stream: true,
  });
  
  for await (const chunk of stream) {
    if (!firstTokenTime && chunk.choices[0]?.delta?.content) {
      firstTokenTime = Date.now() - start;
    }
    if (chunk.choices[0]?.delta?.content) {
      tokenCount++;
    }
  }
  
  const totalTime = Date.now() - start;
  const generationTime = totalTime - firstTokenTime;
  const tps = Math.round(tokenCount / (generationTime / 1000));
  
  console.log(`Model: ${model}`);
  console.log(`TTFT: ${firstTokenTime}ms`);
  console.log(`TPS: ${tps} tokens/sec`);
  console.log(`Total: ${totalTime}ms`);
}

// Benchmark multiple models
const models = ['claude-opus-4-6', 'claude-sonnet-4-5', 'gemini-2.5-flash'];
const prompt = 'Explain the difference between async and parallel programming in 200 words.';

for (const model of models) {
  await measureLatency(model, prompt);
  await new Promise(r => setTimeout(r, 1000)); // Rate limit buffer
}