{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/benchmarks/surveybench", "release": "2026-11-19-3608133e8331", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1670.0, "ci_low": 1580.3, "ci_high": 1807.9, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 1475.7, "ci_low": 1401.6, "ci_high": 1580.0, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1439.4, "ci_low": 1391.8, "ci_high": 1508.2, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1429.3, "ci_low": 1364.2, "ci_high": 1491.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1416.6, "ci_low": 1368.1, "ci_high": 1477.9, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 1380.9, "ci_low": 1317.6, "ci_high": 1464.2, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 1361.8, "ci_low": 1287.3, "ci_high": 1462.2, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 1344.5, "ci_low": 1283.8, "ci_high": 1428.8, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1233.4, "ci_low": 1193.8, "ci_high": 1290.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1217.3, "ci_low": 1159.4, "ci_high": 1293.9, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1133.0, "ci_low": 1072.2, "ci_high": 1221.6, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1118.8, "ci_low": 1045.0, "ci_high": 1196.4, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 1096.2, "ci_low": 997.3, "ci_high": 1190.4, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1071.5, "ci_low": 975.0, "ci_high": 1156.1, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 1053.8, "ci_low": 960.7, "ci_high": 1161.8, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 937.3, "ci_low": 830.4, "ci_high": 1019.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 928.4, "ci_low": 862.9, "ci_high": 989.2, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 845.2, "ci_low": 760.5, "ci_high": 922.3, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 769.8, "ci_low": 715.8, "ci_high": 811.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 743.8, "ci_low": 687.1, "ci_high": 795.2, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 734.9, "ci_low": 630.9, "ci_high": 816.7, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 637.4, "ci_low": 579.5, "ci_high": 689.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 628.8, "ci_low": 499.1, "ci_high": 727.9, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 604.6, "ci_low": 462.0, "ci_high": 727.9, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 577.5, "ci_low": 494.7, "ci_high": 641.0, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 560.5, "ci_low": 482.1, "ci_high": 617.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 427.6, "ci_low": 219.6, "ci_high": 567.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 421.7, "ci_low": 239.4, "ci_high": 529.2, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 96.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 91.74, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 90.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 90.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 89.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 88.34, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 87.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 86.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 83.72, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 83.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 77.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 71.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 64.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 64.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 59.18, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 56.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 39.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 38.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 28.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.37, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 95.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 95.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 94.03, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 93.61, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 93.53, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 92.84, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 92.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 92.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 92.01, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 91.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 91.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 90.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 88.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 87.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 84.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 82.93, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 81.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 72.66, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 53.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 96.14, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 91.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 90.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 88.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 88.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 88.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 85.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 84.88, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 84.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 79.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 79.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 74.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 66.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 63.77, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 56.82, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 56.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 53.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 15.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 13.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 99.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 98.04, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 63.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 98.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 98.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 97.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 93.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 92.75, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 91.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 89.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 84.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 78.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 78.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.01, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 62.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 60.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 59.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 75.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 75.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 70.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 68.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 67.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 60.87, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 59.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 57.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 56.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 56.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 55.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 54.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 52.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 51.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 48.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 47.12, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 40.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 36.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 35.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 31.75, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 29.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 24.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 20.31, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 14.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0088, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.01398, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01848, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01977, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.02133, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.02156, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.03727, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.05686, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.06352, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.06934, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.07298, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0924, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.09428, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.12448, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.13101, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.1429, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.20369, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.3092, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.33324, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.33647, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.3542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.36596, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.45707, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.74209, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.79488, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.51051, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 73.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 74.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 91.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 92.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 104.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 130.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 133.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 137.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 147.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 169.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 180.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 193.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 221.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 252.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 266.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 337.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 351.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 389.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 434.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 435.9, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 439.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 690.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1956.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 5888.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 8112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 9187.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 14333.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 21981.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 26601.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 27760.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 30893.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33902.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 39999.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 40298.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 41542.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 50889.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 51553.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 57343.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63237.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70935.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen4-max", "model_name": "Qwen4-Max", "developer": "Alibaba", "configuration": "Qwen4-Max", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-16", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-hydrogen", "model_name": "Gemini 4 Hydrogen", "developer": "Google", "configuration": "Gemini 4 Hydrogen", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-astra", "model_name": "GPT-6.2 Astra", "developer": "OpenAI", "configuration": "GPT-6.2 Astra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-18", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-2-sol", "model_name": "GPT-6.2 Sol", "developer": "OpenAI", "configuration": "GPT-6.2 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-17", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-carbon", "model_name": "Gemini 4 Carbon", "developer": "Google", "configuration": "Gemini 4 Carbon", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-ember", "model_name": "Muse Ember", "developer": "Meta", "configuration": "Muse Ember", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-neon", "model_name": "Gemini 4 Neon", "developer": "Google", "configuration": "Gemini 4 Neon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}]}