{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/data", "release": "2026-11-19-3608133e8331", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 892.0, "ci_low": 641.0, "ci_high": 1081.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 780.0, "ci_low": 589.0, "ci_high": 940.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 755.0, "ci_low": 535.0, "ci_high": 964.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 727.0, "ci_low": 567.0, "ci_high": 890.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 616.0, "ci_low": 466.0, "ci_high": 761.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 616.0, "ci_low": 470.0, "ci_high": 757.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 614.0, "ci_low": 470.0, "ci_high": 746.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 614.0, "ci_low": 470.0, "ci_high": 766.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 614.0, "ci_low": 429.0, "ci_high": 742.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 614.0, "ci_low": 429.0, "ci_high": 742.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 584.0, "ci_low": 327.0, "ci_high": 806.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 572.0, "ci_low": 419.0, "ci_high": 722.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 572.0, "ci_low": 428.0, "ci_high": 718.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 572.0, "ci_low": 412.0, "ci_high": 727.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 571.0, "ci_low": 420.0, "ci_high": 699.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 571.0, "ci_low": 412.0, "ci_high": 721.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 571.0, "ci_low": 428.0, "ci_high": 698.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 555.0, "ci_low": 263.0, "ci_high": 727.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 529.0, "ci_low": 404.0, "ci_high": 643.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 529.0, "ci_low": 388.0, "ci_high": 670.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 529.0, "ci_low": 373.0, "ci_high": 653.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 529.0, "ci_low": 382.0, "ci_high": 670.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 529.0, "ci_low": 361.0, "ci_high": 669.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 529.0, "ci_low": 308.0, "ci_high": 680.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 529.0, "ci_low": 407.0, "ci_high": 643.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 529.0, "ci_low": 382.0, "ci_high": 670.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 529.0, "ci_low": 382.0, "ci_high": 670.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 491.0, "ci_low": 0.0, "ci_high": 727.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 489.0, "ci_low": 376.0, "ci_high": 623.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 489.0, "ci_low": 363.0, "ci_high": 621.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 453.0, "ci_low": 346.0, "ci_high": 567.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 420.0, "ci_low": 367.0, "ci_high": 488.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 409.0, "ci_low": 191.0, "ci_high": 598.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 310.0, "ci_low": 0.0, "ci_high": 641.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 249.0, "ci_low": 0.0, "ci_high": 486.0, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 439.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 425.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 197.0, "ci_low": 0.0, "ci_high": 404.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 33.0, "ci_low": 0.0, "ci_high": 248.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 248.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 272.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 368.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 421.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 486.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 511.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 512.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 515.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 517.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 523.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 528.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 528.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 534.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 543.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 552.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 569.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 586.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 590.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 594.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 611.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 615.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 623.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 626.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 636.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 647.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 651.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 688.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 713.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 732.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 767.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 818.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 826.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 851.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 882.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 886.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 943.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 1024.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 1048.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 1054.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 1178.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 1241.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 1261.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1279.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1370.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 1385.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1502.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 1520.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1844.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 2083.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.27, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 4.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 5.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.000129, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.000215, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.000253, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.00028, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.000332, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.00045, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.000615, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 0.000635, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.000658, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 0.000789, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.000815, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.000869, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.000881, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.000948, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 0.001003, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.001023, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.001035, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.001253, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.001382, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.001774, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.001849, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.001925, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.002391, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 0.002494, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.002502, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.003521, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.003997, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.004053, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.004091, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.004995, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.005835, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.006417, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.006979, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.007186, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.007678, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 0.008041, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.009534, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.011659, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.013619, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.021301, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.021951, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 0.02623, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.029713, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.049138, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.158258, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.165374, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1147.0, "ci_low": 979.0, "ci_high": 1373.0, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 1126.0, "ci_low": 949.0, "ci_high": 1347.0, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1100.0, "ci_low": 789.0, "ci_high": 1450.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1052.0, "ci_low": 770.0, "ci_high": 1302.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1000.0, "ci_low": 765.0, "ci_high": 1282.0, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 977.0, "ci_low": 746.0, "ci_high": 1172.0, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 946.0, "ci_low": 604.0, "ci_high": 1284.0, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 927.0, "ci_low": 657.0, "ci_high": 1152.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 848.0, "ci_low": 577.0, "ci_high": 1079.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 839.0, "ci_low": 595.0, "ci_high": 1072.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 707.0, "ci_low": 458.0, "ci_high": 933.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 683.0, "ci_low": 487.0, "ci_high": 1130.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 683.0, "ci_low": 487.0, "ci_high": 1130.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 645.0, "ci_low": 0.0, "ci_high": 890.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 555.0, "ci_low": 392.0, "ci_high": 727.0, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 555.0, "ci_low": 396.0, "ci_high": 765.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 491.0, "ci_low": 299.0, "ci_high": 659.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 358.0, "ci_low": 0.0, "ci_high": 619.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 358.0, "ci_low": 0.0, "ci_high": 600.0, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 519.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 789.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 837.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1297.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1937.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 2112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 3673.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 3722.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 4020.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 4039.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 4042.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 4783.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 4858.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 5365.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 5373.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5612.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 6172.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 6219.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 6591.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 6979.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 7106.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 7116.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 7513.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 8122.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 8591.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 12.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.53, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.007164, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.008238, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.008653, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01643, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.026106, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.031778, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.039953, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.053448, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.070617, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 0.073008, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.076745, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.079469, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.115847, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.12185, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.133163, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.162683, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.175778, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.207138, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.267174, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.284697, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.307958, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.348513, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.384659, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.523922, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1.098534, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1068.0, "ci_low": 888.0, "ci_high": 1241.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 928.0, "ci_low": 676.0, "ci_high": 1184.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 912.0, "ci_low": 697.0, "ci_high": 1087.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 889.0, "ci_low": 593.0, "ci_high": 1229.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 839.0, "ci_low": 541.0, "ci_high": 1130.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 810.0, "ci_low": 582.0, "ci_high": 1006.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 780.0, "ci_low": 586.0, "ci_high": 956.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon (low reasoning)", "value": 766.0, "ci_low": 428.0, "ci_high": 972.0, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 628.0, "ci_low": 429.0, "ci_high": 849.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 614.0, "ci_low": 470.0, "ci_high": 766.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 614.0, "ci_low": 470.0, "ci_high": 746.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 584.0, "ci_low": 435.0, "ci_high": 749.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 572.0, "ci_low": 428.0, "ci_high": 717.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 572.0, "ci_low": 412.0, "ci_high": 727.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 572.0, "ci_low": 428.0, "ci_high": 708.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 572.0, "ci_low": 412.0, "ci_high": 727.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 572.0, "ci_low": 429.0, "ci_high": 720.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 571.0, "ci_low": 420.0, "ci_high": 699.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 571.0, "ci_low": 434.0, "ci_high": 727.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 571.0, "ci_low": 428.0, "ci_high": 724.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 540.0, "ci_low": 296.0, "ci_high": 765.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 529.0, "ci_low": 409.0, "ci_high": 642.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 529.0, "ci_low": 388.0, "ci_high": 670.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 529.0, "ci_low": 373.0, "ci_high": 653.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 529.0, "ci_low": 329.0, "ci_high": 698.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 529.0, "ci_low": 263.0, "ci_high": 688.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 529.0, "ci_low": 393.0, "ci_high": 652.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 529.0, "ci_low": 404.0, "ci_high": 671.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 491.0, "ci_low": 230.0, "ci_high": 714.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 491.0, "ci_low": 187.0, "ci_high": 738.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 491.0, "ci_low": 0.0, "ci_high": 727.0, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 491.0, "ci_low": 299.0, "ci_high": 659.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol (low reasoning)", "value": 491.0, "ci_low": 230.0, "ci_high": 714.0, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 491.0, "ci_low": 230.0, "ci_high": 714.0, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 490.0, "ci_low": 381.0, "ci_high": 600.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 489.0, "ci_low": 376.0, "ci_high": 608.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 489.0, "ci_low": 363.0, "ci_high": 621.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 453.0, "ci_low": 355.0, "ci_high": 538.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 449.0, "ci_low": 266.0, "ci_high": 608.0, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 445.0, "ci_low": 135.0, "ci_high": 626.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 445.0, "ci_low": 215.0, "ci_high": 639.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 409.0, "ci_low": 100.0, "ci_high": 609.0, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 385.0, "ci_low": 276.0, "ci_high": 464.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 385.0, "ci_low": 279.0, "ci_high": 459.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 367.0, "ci_low": 126.0, "ci_high": 564.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 358.0, "ci_low": 0.0, "ci_high": 619.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 358.0, "ci_low": 0.0, "ci_high": 595.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 358.0, "ci_low": 0.0, "ci_high": 600.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 323.0, "ci_low": 51.0, "ci_high": 529.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 323.0, "ci_low": 0.0, "ci_high": 528.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 310.0, "ci_low": 0.0, "ci_high": 575.0, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (minimal reasoning)", "value": 310.0, "ci_low": 0.0, "ci_high": 595.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 249.0, "ci_low": 0.0, "ci_high": 453.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 445.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 197.0, "ci_low": 0.0, "ci_high": 439.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 443.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 147.0, "ci_low": 0.0, "ci_high": 443.0, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 115.0, "ci_low": 0.0, "ci_high": 485.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 273.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 276.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 367.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 423.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 488.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 508.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 518.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 519.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 522.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 524.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 526.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 534.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 539.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 540.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 553.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 565.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 570.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 587.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 592.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 595.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 598.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 612.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 620.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 625.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 635.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 647.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 681.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 713.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 725.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 792.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 822.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 876.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 916.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 917.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 1061.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 1194.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 1198.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1241.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 1394.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 1440.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 1467.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 1573.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 1580.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 1583.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 1695.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 1707.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1710.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (minimal reasoning)", "value": 1795.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1813.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 1885.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1966.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol (low reasoning)", "value": 2025.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 2197.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 2273.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 2298.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon (low reasoning)", "value": 2429.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 2579.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (low reasoning)", "value": 2760.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 2799.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 2847.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 2920.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 2927.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 2958.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 2989.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 3009.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 3026.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 3090.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 3149.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 3219.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 3334.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 3388.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 3433.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 3622.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 4184.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 4232.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max (low reasoning)", "value": 4402.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 5040.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 41.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 41.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (minimal reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.31, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 3.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 7.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "parallel-decisions-fast", "model_name": "Parallel Decisions Fast", "developer": "Parallel", "configuration": "Parallel Decisions Fast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "respan-span-02", "model_name": "Span-02", "developer": "Respan", "configuration": "Span-02", "value": 0.000163, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-flash-v1-2", "model_name": "Decider Flash v1.2", "developer": "Perplexity", "configuration": "Decider Flash v1.2", "value": 0.000273, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-joey-1", "model_name": "Joey 1", "developer": "Baseten", "configuration": "Joey 1", "value": 0.000279, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-2", "model_name": "Decider v1.2", "developer": "Perplexity", "configuration": "Decider v1.2", "value": 0.000301, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1-1", "model_name": "d1.1", "developer": "Liquid AI", "configuration": "d1.1", "value": 0.000318, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1-ultra", "model_name": "d1 Ultra", "developer": "Liquid AI", "configuration": "d1 Ultra", "value": 0.000334, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "morph-reflex", "model_name": "Reflex", "developer": "Morph", "configuration": "Reflex", "value": 0.000589, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-v1-5", "model_name": "Drex v1.5", "developer": "Nace AI", "configuration": "Drex v1.5", "value": 0.000712, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.000733, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-elephant-1", "model_name": "Elephant 1", "developer": "Baseten", "configuration": "Elephant 1", "value": 0.000835, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "microsoft-microsoft-decision-1", "model_name": "Microsoft Decision 1", "developer": "Microsoft", "configuration": "Microsoft Decision 1", "value": 0.000864, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-v2-0", "model_name": "Drex v2.0", "developer": "Nace AI", "configuration": "Drex v2.0", "value": 0.0009, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.000964, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-max-finance", "model_name": "Drex Max Finance", "developer": "Nace AI", "configuration": "Drex Max Finance", "value": 0.001017, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "nace-ai-drex-max", "model_name": "Drex Max", "developer": "Nace AI", "configuration": "Drex Max", "value": 0.001103, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.001314, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-14", "model_name": "Jev 1.14", "developer": "TypeSafe", "configuration": "Jev 1.14", "value": 0.001387, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 0.001584, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 0.001624, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "baseten-mammoth-1", "model_name": "Mammoth 1", "developer": "Baseten", "configuration": "Mammoth 1", "value": 0.001741, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 0.001915, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.00195, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-decision-1", "model_name": "Gemini Decision 1", "developer": "Google", "configuration": "Gemini Decision 1", "value": 0.00234, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.002546, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.002591, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.003021, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 0.003208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.003586, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (minimal reasoning)", "value": 0.003782, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.003857, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 0.004732, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-omni", "model_name": "Clef Omni", "developer": "Cloudflare", "configuration": "Clef Omni", "value": 0.004935, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 0.005024, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.005627, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.005796, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.006058, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.006825, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.007487, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember (low reasoning)", "value": 0.010603, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.010842, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.011273, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 0.014236, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max (low reasoning)", "value": 0.015033, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.015378, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.015582, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.016258, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.016514, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon (low reasoning)", "value": 0.016722, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.017217, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 0.017541, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.020513, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.020984, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 0.021951, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 0.024288, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol (low reasoning)", "value": 0.026824, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.028933, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 0.029717, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.030805, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.035407, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.041972, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 0.044593, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 0.053958, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen (low reasoning)", "value": 0.065383, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 0.066301, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.067356, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.074801, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon (low reasoning)", "value": 0.078987, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 0.079975, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.094311, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.094843, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.102155, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 0.103691, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.336607, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.35722, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 80.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 79.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 77.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 77.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 71.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 69.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 65.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 63.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63.09, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 61.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 56.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 40.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 29.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 12.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 71.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 60.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 53.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 53.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 39.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 35.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 28.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.27, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 98.23, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 97.73, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 97.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 97.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 97.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 96.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 96.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 96.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 95.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 95.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 94.39, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 94.18, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 90.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.012, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.03, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.137, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.226, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.244, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.292, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.363, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.393, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.411, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.417, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.446, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.536, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.661, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.696, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.708, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.786, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.905, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.976, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.083, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.179, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.185, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.202, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.226, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.702, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.744, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.071, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 2.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 11.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 12.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 22.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 39.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 48.81, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 24.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 30.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 38.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 79.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 76.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 72.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 54.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 52.97, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 47.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 42.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 30.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 30.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 25.59, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 69.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 69.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 35.71, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 33.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 98.85, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 98.51, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 98.39, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 98.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 98.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 97.85, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 97.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 97.32, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 97.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.08, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 95.87, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.42, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 89.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 16.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.006, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.054, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.065, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.345, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.357, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.458, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.601, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.637, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 1.101, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1.268, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 1.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.655, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.982, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 2.101, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 2.131, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2.357, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 2.464, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 2.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.44, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.548, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 3.643, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 3.863, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 4.006, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 4.524, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 5.321, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 6.738, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 82.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 28.89, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 23.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 39.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 48.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 57.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 59.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 60.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 63.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 65.48, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 70.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 70.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 76.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 79.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 80.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 83.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 84.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 85.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 97.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 22.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 36.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 42.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 51.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-11", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 98.45, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 97.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 97.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.84, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 96.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 96.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 96.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 95.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 95.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 95.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.06, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 93.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 93.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 92.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 90.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 88.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 61.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 95.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 95.13, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 95.13, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 94.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 92.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 92.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 91.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 91.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 91.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 91.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 90.27, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 86.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 59.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.51, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.96, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 1.71, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 2.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 2.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 2.88, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.54, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 5.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 84.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 98.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.000477, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.001015, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.001172, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.001208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.001542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.00204, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.002047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.003277, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.00356, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.005035, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.00607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.006138, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.006853, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.007011, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.008005, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.008268, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.008384, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.008518, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.010292, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.010921, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.012462, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.016418, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.017191, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.021347, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.021933, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.023719, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.025572, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.034526, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.035464, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.036073, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.03712, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.040006, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.045438, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.046895, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.065278, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.087539, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.099128, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 4.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 6.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 6.7, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 7.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 8.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 9.7, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 9.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 10.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 12.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 17.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 17.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 18.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 20.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 20.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 21.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 23.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 25.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 32.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 36.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 41.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 45.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 62.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 66.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 78.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 97.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 114.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 426.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 442.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 457.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1226.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1227.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1442.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1600.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2180.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 2416.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2890.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3014.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3651.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 3816.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 3939.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3996.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1563.4, "ci_low": 1477.8, "ci_high": 1692.1, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1551.3, "ci_low": 1433.6, "ci_high": 1729.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 1437.8, "ci_low": 1370.3, "ci_high": 1544.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1403.1, "ci_low": 1311.4, "ci_high": 1511.6, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1375.1, "ci_low": 1304.9, "ci_high": 1463.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1370.0, "ci_low": 1276.9, "ci_high": 1493.4, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1333.6, "ci_low": 1249.2, "ci_high": 1445.1, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1322.9, "ci_low": 1245.0, "ci_high": 1405.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1280.2, "ci_low": 1195.9, "ci_high": 1389.8, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1266.0, "ci_low": 1176.5, "ci_high": 1366.9, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1256.1, "ci_low": 1190.5, "ci_high": 1332.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1252.4, "ci_low": 1193.5, "ci_high": 1332.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 1241.7, "ci_low": 1173.4, "ci_high": 1325.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1223.2, "ci_low": 1095.4, "ci_high": 1329.1, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1203.8, "ci_low": 1146.9, "ci_high": 1259.5, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1194.4, "ci_low": 1147.5, "ci_high": 1276.7, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1187.9, "ci_low": 1081.9, "ci_high": 1318.2, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 1142.7, "ci_low": 1065.8, "ci_high": 1226.2, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1120.1, "ci_low": 1050.0, "ci_high": 1187.7, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1086.0, "ci_low": 983.0, "ci_high": 1185.4, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1078.8, "ci_low": 983.4, "ci_high": 1175.5, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1046.2, "ci_low": 943.6, "ci_high": 1120.4, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1042.6, "ci_low": 950.4, "ci_high": 1135.4, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1025.3, "ci_low": 953.9, "ci_high": 1101.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 1013.2, "ci_low": 879.2, "ci_high": 1112.5, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 991.9, "ci_low": 851.0, "ci_high": 1114.3, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 951.3, "ci_low": 882.5, "ci_high": 1024.6, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 809.3, "ci_low": 677.9, "ci_high": 919.0, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 806.7, "ci_low": 747.6, "ci_high": 853.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 771.4, "ci_low": 706.4, "ci_high": 824.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 726.2, "ci_low": 632.0, "ci_high": 814.4, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 715.6, "ci_low": 643.5, "ci_high": 787.3, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 645.7, "ci_low": 541.4, "ci_high": 737.7, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 605.2, "ci_low": 494.1, "ci_high": 694.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 594.8, "ci_low": 472.3, "ci_high": 692.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 585.1, "ci_low": 473.0, "ci_high": 661.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 530.0, "ci_low": 390.6, "ci_high": 611.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 95.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 88.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 87.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 86.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 83.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 83.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 81.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 80.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 78.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 78.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 77.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 76.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 71.96, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 71.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 68.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 67.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 67.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 66.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 61.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 61.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 60.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 59.77, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 58.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 56.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 56.05, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 55.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 54.37, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 51.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 47.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 46.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 43.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 40.87, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 30.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 89.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 85.37, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 84.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 82.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 79.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 76.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 70.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 69.77, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 69.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 64.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 60.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 60.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 59.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 51.37, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 48.48, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 47.46, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 46.01, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 45.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.59, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 43.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 42.47, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 41.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 41.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 39.39, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 37.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 35.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 31.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 29.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 28.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 24.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 24.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.00857, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.00865, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.00905, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01042, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.01122, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.01162, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01262, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.01477, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.02552, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0359, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0448, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.04607, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.04626, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0465, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.06134, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.06722, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.07745, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.11468, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.1355, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.14527, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.15273, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.15295, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.16376, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.17891, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.18211, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.19567, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.20966, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.2498, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.25243, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.26102, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.4537, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.54068, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.77156, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.80661, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.09215, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.39571, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 15.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 28.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 30.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 44.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 51.7, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 67.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 73.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 78.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 81.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 83.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 83.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 89.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 93.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 98.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 114.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 134.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 154.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 157.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 169.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 174.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 178.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 202.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 217.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 228.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 285.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 311.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 330.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 332.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 374.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 424.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 437.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 488.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 543.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1288.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 4453.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 7306.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 12086.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 15626.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16952.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 21019.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22094.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 27796.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 31368.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 32448.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33285.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33765.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 36572.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 45141.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 48462.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 51739.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 61.27, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 58.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 57.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 56.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 53.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 52.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 51.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 51.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 51.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 51.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 51.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 47.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 46.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 45.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 45.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 44.06, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 43.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 43.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 41.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 40.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 38.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 37.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 36.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 28.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 27.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 26.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 25.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 25.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 24.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 24.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 17.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 15.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.9615, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.7174, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.7138, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.7039, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.6973, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6872, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.6834, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.6768, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.6747, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.6574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.6474, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6473, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6472, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.547, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.5458, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.5418, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.5275, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.5233, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.5124, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.5024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.5002, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.4478, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.4426, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.4316, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.4192, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.413, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.2944, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.2574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.2259, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.1435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.1005, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0943, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0313, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0116, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0097, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -0.1652, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -0.2137, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1032894.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 965271.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 911566.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 858022.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 842431.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 789365.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 775520.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 740355.71, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 738878.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 701058.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 699863.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 695680.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 693379.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 684623.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 673560.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 654270.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 621558.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 600356.58, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 599139.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 585571.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 571202.72, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 547930.08, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 539202.03, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 533401.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 507735.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 489176.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 459835.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 337250.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 301288.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 187636.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 156831.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 104867.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 99775.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32146.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 11834.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 9717.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -159881.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -205000.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 72.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 60.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 59.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 57.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 57.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 53.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 52.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 52.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 52.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.96, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 50.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 48.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 47.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 43.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 42.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 41.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 40.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 39.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 38.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 38.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 35.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 34.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 31.85, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 25.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 23.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 17.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 15.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 13.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 12.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 12.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 55.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 55.29, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 55.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 55.01, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 54.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 54.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 54.96, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 54.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 54.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 54.87, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 54.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 54.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 54.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 54.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 54.82, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 54.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 54.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.66, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 54.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 54.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 54.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 54.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 54.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 54.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 54.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 54.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 53.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 53.97, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 53.97, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 53.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 51.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 69.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 69.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 68.77, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 68.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 68.49, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 68.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 68.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 68.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 67.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 67.13, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 66.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 66.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 65.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 65.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 65.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 65.13, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 63.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 61.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 60.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 55.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 51.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 51.27, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 50.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 48.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 45.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 44.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 43.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 39.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 30.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 28.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 48.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 44.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 44.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 42.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 42.75, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 38.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 38.08, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 36.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 36.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 35.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 34.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 31.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 30.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 29.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 28.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 28.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 27.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 26.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 25.92, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 25.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 20.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 17.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 15.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 14.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 14.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.58, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 9.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 9.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.014182, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.023373, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.029678, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.029807, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.031284, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.037854, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.038977, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.053281, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.065733, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.079867, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.11103, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.131184, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.148765, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.158208, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.16178, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.183125, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.187194, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.197608, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.229739, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.242268, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.244047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.420282, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.42358, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.436569, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.518802, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.52247, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.567512, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.690881, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.73084, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.768818, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.012818, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.064612, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.225685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 1.282275, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.60214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.759152, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.48607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1670.0, "ci_low": 1580.3, "ci_high": 1807.9, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1475.7, "ci_low": 1401.6, "ci_high": 1580.0, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1439.4, "ci_low": 1391.8, "ci_high": 1508.2, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1429.3, "ci_low": 1364.2, "ci_high": 1491.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1416.6, "ci_low": 1368.1, "ci_high": 1477.9, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1380.9, "ci_low": 1317.6, "ci_high": 1464.2, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1361.8, "ci_low": 1287.3, "ci_high": 1462.2, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 1344.5, "ci_low": 1283.8, "ci_high": 1428.8, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1233.4, "ci_low": 1193.8, "ci_high": 1290.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1217.3, "ci_low": 1159.4, "ci_high": 1293.9, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1133.0, "ci_low": 1072.2, "ci_high": 1221.6, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1118.8, "ci_low": 1045.0, "ci_high": 1196.4, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1096.2, "ci_low": 997.3, "ci_high": 1190.4, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1071.5, "ci_low": 975.0, "ci_high": 1156.1, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1053.8, "ci_low": 960.7, "ci_high": 1161.8, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 937.3, "ci_low": 830.4, "ci_high": 1019.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 928.4, "ci_low": 862.9, "ci_high": 989.2, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 845.2, "ci_low": 760.5, "ci_high": 922.3, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 769.8, "ci_low": 715.8, "ci_high": 811.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 743.8, "ci_low": 687.1, "ci_high": 795.2, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 734.9, "ci_low": 630.9, "ci_high": 816.7, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 637.4, "ci_low": 579.5, "ci_high": 689.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 628.8, "ci_low": 499.1, "ci_high": 727.9, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 604.6, "ci_low": 462.0, "ci_high": 727.9, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 577.5, "ci_low": 494.7, "ci_high": 641.0, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 560.5, "ci_low": 482.1, "ci_high": 617.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 427.6, "ci_low": 219.6, "ci_high": 567.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 421.7, "ci_low": 239.4, "ci_high": 529.2, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 96.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 91.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 90.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 90.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 89.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 88.34, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 87.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 86.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 83.72, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 83.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 77.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 71.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 64.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 64.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 59.18, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 56.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 39.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 38.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 28.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.37, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 95.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 95.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 94.03, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 93.61, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 93.53, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 92.84, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 92.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 92.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 92.01, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 91.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 91.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 90.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 88.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 87.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 84.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 82.93, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 81.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 72.66, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 53.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 96.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 91.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 90.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 88.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 88.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 88.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 85.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 84.88, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 84.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 79.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 79.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 74.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 66.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 63.77, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 56.82, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 56.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 53.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 15.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 13.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 99.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 98.04, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 63.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 98.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 97.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 93.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 92.75, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 91.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 89.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 84.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 78.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 78.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.01, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 62.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 60.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 59.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 75.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 75.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 70.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 68.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 67.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 60.87, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 59.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 57.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 56.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 56.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 55.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 54.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 52.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 51.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 48.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 47.12, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 40.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 36.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 35.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 31.75, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 29.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 24.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 20.31, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 14.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0088, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.01398, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01848, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01977, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.02133, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.02156, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.03727, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.05686, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.06352, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.06934, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.07298, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0924, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.09428, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.12448, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.13101, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.1429, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.20369, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.3092, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.33324, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.33647, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.3542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.36596, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.45707, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.74209, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.79488, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.51051, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 73.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 74.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 91.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 92.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 104.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 130.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 133.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 137.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 147.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 169.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 180.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 193.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 221.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 252.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 266.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 337.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 351.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 389.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 434.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 435.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 439.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 690.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1956.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 5888.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 8112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 9187.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 14333.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 21981.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 26601.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 27760.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 30893.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33902.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 39999.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 40298.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 41542.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50889.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 51553.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 57343.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63237.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70935.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}]}