Gemini 2.5 Flash vs GPT-4.1 Nano Vision: Count Shapes and Report a Timeout

import json
import os
import time
from decimal import Decimal as D
import requests

BASE = "https://api.crazyrouter.com/v1"

def client():
    s = requests.Session()
    s.trust_env = False
    return s

def catalog(wanted):
    with client() as s:
        r = s.get(BASE + "/models", headers={"Authorization": "Bearer " + os.environ["CRAZYROUTER_API_KEY"]}, timeout=(10, 40), allow_redirects=False)
        r.raise_for_status()
        names = {x["id"] for x in r.json()["data"]}
    return {name: name in names for name in wanted}

def request(model, prompt, mode="chat", **options):
    path = {"chat": "/chat/completions", "messages": "/messages", "responses": "/responses"}[mode]
    payload = {"model": model, **options}
    if mode == "responses":
        payload["input"] = prompt
    else:
        payload["messages"] = [{"role": "user", "content": prompt}]
    headers = {"Authorization": "Bearer " + os.environ["CRAZYROUTER_API_KEY"]}
    if mode == "messages":
        headers = {"x-api-key": os.environ["CRAZYROUTER_API_KEY"], "anthropic-version": "2023-06-01"}
    rec = {"model": model, "endpoint": BASE + path, "request": payload}
    start = time.perf_counter()
    try:
        with client() as s:
            r = s.post(BASE + path, headers=headers, json=payload, timeout=(10, 90), allow_redirects=False)
        rec["http"] = r.status_code
        if r.status_code == 200:
            data = r.json()
            rec.update(response_id=data.get("id"), usage=data.get("usage", {}))
            if mode == "messages":
                output = "".join(x.get("text", "") for x in data.get("content", []) if x.get("type") == "text")
                complete = data.get("stop_reason") == "end_turn"
                finish = data.get("stop_reason")
            elif mode == "responses":
                output = "".join(c.get("text", "") for item in data.get("output", []) if item.get("type") == "message" for c in item.get("content", []) if c.get("type") == "output_text")
                complete = data.get("status") == "completed"
                finish = data.get("status")
            else:
                choice = data["choices"][0]
                output = choice["message"].get("content") or ""
                complete = choice.get("finish_reason") == "stop"
                finish = choice.get("finish_reason")
            rec.update(output=output, complete=complete, finish=finish)
        else:
            rec["error"] = "HTTP_" + str(r.status_code)
    except (requests.RequestException, ValueError, KeyError, IndexError) as error:
        rec["error"] = type(error).__name__
    rec["seconds"] = round(time.perf_counter() - start, 3)
    return rec

def accepted(rec, expected):
    if not rec.get("complete"):
        return False
    try:
        return json.loads(rec["output"]) == expected
    except (ValueError, TypeError):
        return False

def emit(**data):
    print(json.dumps(data, ensure_ascii=False))

import base64
import hashlib
import io
from PIL import Image, ImageDraw

def fixture(red_count, blue_count):
    image = Image.new("RGB", (480, 240), "white")
    draw = ImageDraw.Draw(image)
    for i in range(red_count):
        draw.rectangle((20+i*100, 20, 75+i*100, 75), fill=(220, 20, 40))
    for i in range(blue_count):
        draw.ellipse((20+i*100, 140, 75+i*100, 195), fill=(20, 50, 220))
    buffer = io.BytesIO()
    image.save(buffer, format="PNG")
    return buffer.getvalue()

models = ["gemini-2.5-flash", "gpt-4.1-nano"]
available = catalog(models)
tests = []
rows = []
fixtures = []
for red, blue in [(2, 3), (3, 1)]:
    data = fixture(red, blue)
    image = Image.open(io.BytesIO(data))
    assert image.size == (480, 240) and image.getpixel((25, 25)) == (220, 20, 40)
    tests.append({"case": f"fixture {red} red / {blue} blue PNG", "passed": True})
    fixtures.append({"red_squares": red, "blue_circles": blue, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)})
    for model in models:
        if not available[model]:
            continue
        prompt = [{"type": "text", "text": 'Count the visible shapes. Return only a JSON object with integer keys red_squares and blue_circles. No Markdown.'},
                  {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(data).decode()}}]
        rec = request(model, prompt, max_tokens=1024)
        rec.update(case=f"{red} red / {blue} blue", passed=accepted(rec, {"red_squares": red, "blue_circles": blue}), fixture_sha256=hashlib.sha256(data).hexdigest())
        rows.append(rec)
emit(local_tests=tests, catalog=available, requests=rows, fixtures=fixtures,
     scope="Two synthetic images per route; no OCR, PDF, video, real-world visual accuracy, or latency percentile benchmark")