{
  "$schema": "https://schema.org/Dataset",
  "name": "CodeReviewBench leaderboard",
  "description": "How well AI models find real bugs in code review, measured on merged pull requests from production open-source repositories.",
  "url": "https://codereviewbench.com/leaderboard",
  "license": "https://github.com/kodustech/codereviewbench",
  "measuredOn": {
    "pullRequests": 30,
    "confirmedBugs": 121,
    "goldSet": "gold-v3",
    "judge": [
      "Claude Haiku 5.5"
    ],
    "repositories": [
      "cal.com",
      "discourse",
      "grafana",
      "keycloak",
      "sentry"
    ],
    "languages": [
      "go",
      "java",
      "python",
      "ruby",
      "typescript"
    ]
  },
  "generatedAt": "2026-11-09T23:02:28.320Z",
  "pricesCollectedAt": "2026-11-07",
  "metricDefinitions": {
    "recall": "Valid true positives found / all valid true positives in the gold set, micro-averaged.",
    "precision": "Published comments matching a valid true positive / all published comments. Each true positive counts once.",
    "f1": "Harmonic mean of recall and precision.",
    "f2": "F-score with beta = 2: 5*P*R / (4*P + R). Recall weighs twice precision. Ranking metric within each mode.",
    "costPerPR": "Token cost at vendor list price with prompt cache, divided by pull requests.",
    "costPerBugFound": "Same total cost divided by true positives found."
  },
  "modes": {
    "balanced": "The normal review: one generalist agent.",
    "heavy": "Generalist plus 9 specialist lens agents in parallel, then the same verify/dedup steps."
  },
  "models": [
    {
      "id": "kimi-k3.1-pro",
      "name": "Kimi K3.1 Pro",
      "provider": "Moonshot",
      "mode": "balanced",
      "rank": 1,
      "f2": 63.5,
      "recall": 68.6,
      "precision": 49.1,
      "f1": 57.2,
      "bugsFound": 83,
      "bugsTotal": 121,
      "comments": 169,
      "pullRequests": 30,
      "costPerPR": 1.2437,
      "costPerBugFound": 0.4495,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T19:50:38.562Z",
      "url": "https://codereviewbench.com/model/kimi-k3.1-pro"
    },
    {
      "id": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "provider": "OpenAI",
      "mode": "balanced",
      "rank": 2,
      "f2": 45.9,
      "recall": 43,
      "precision": 62.6,
      "f1": 51,
      "bugsFound": 52,
      "bugsTotal": 121,
      "comments": 83,
      "pullRequests": 30,
      "costPerPR": 0.7874,
      "costPerBugFound": 0.4543,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T20:03:22.217Z",
      "url": "https://codereviewbench.com/model/gpt-6.1-sol"
    },
    {
      "id": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "provider": "Anthropic",
      "mode": "balanced",
      "rank": 3,
      "f2": 43.9,
      "recall": 42.1,
      "precision": 52.6,
      "f1": 46.8,
      "bugsFound": 51,
      "bugsTotal": 121,
      "comments": 97,
      "pullRequests": 30,
      "costPerPR": 0.7153,
      "costPerBugFound": 0.4208,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-09T13:17:34.130Z",
      "url": "https://codereviewbench.com/model/claude-opus-5.5"
    },
    {
      "id": "glm-5.3",
      "name": "GLM-5.3",
      "provider": "Zhipu",
      "mode": "balanced",
      "rank": 4,
      "f2": 41.7,
      "recall": 41.3,
      "precision": 43.5,
      "f1": 42.4,
      "bugsFound": 50,
      "bugsTotal": 121,
      "comments": 115,
      "pullRequests": 30,
      "costPerPR": 1.1707,
      "costPerBugFound": 0.7024,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-08T01:30:50.430Z",
      "url": "https://codereviewbench.com/model/glm-5.3"
    },
    {
      "id": "deepseek-v4.1-flash",
      "name": "DeepSeek V4.1 Flash",
      "provider": "DeepSeek",
      "mode": "balanced",
      "rank": 5,
      "f2": 41,
      "recall": 40.5,
      "precision": 43.4,
      "f1": 41.9,
      "bugsFound": 49,
      "bugsTotal": 121,
      "comments": 113,
      "pullRequests": 30,
      "costPerPR": 0.134,
      "costPerBugFound": 0.0821,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T20:00:51.712Z",
      "url": "https://codereviewbench.com/model/deepseek-v4.1-flash"
    },
    {
      "id": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "provider": "Anthropic",
      "mode": "balanced",
      "rank": 6,
      "f2": 40,
      "recall": 37.2,
      "precision": 57,
      "f1": 45,
      "bugsFound": 45,
      "bugsTotal": 121,
      "comments": 79,
      "pullRequests": 30,
      "costPerPR": 0.5416,
      "costPerBugFound": 0.361,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T14:53:34.961Z",
      "url": "https://codereviewbench.com/model/claude-sonnet-5.5"
    },
    {
      "id": "muse-spark-1.2",
      "name": "Muse Spark 1.2",
      "provider": "Meta",
      "mode": "balanced",
      "rank": 7,
      "f2": 36.4,
      "recall": 37.2,
      "precision": 33.6,
      "f1": 35.3,
      "bugsFound": 45,
      "bugsTotal": 121,
      "comments": 134,
      "pullRequests": 30,
      "costPerPR": 0.7249,
      "costPerBugFound": 0.4832,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T16:34:00.953Z",
      "url": "https://codereviewbench.com/model/muse-spark-1.2"
    },
    {
      "id": "glm-5.3-flash",
      "name": "GLM-5.3 Flash",
      "provider": "Zhipu",
      "mode": "balanced",
      "rank": 8,
      "f2": 36.3,
      "recall": 35.5,
      "precision": 39.8,
      "f1": 37.5,
      "bugsFound": 43,
      "bugsTotal": 121,
      "comments": 108,
      "pullRequests": 30,
      "costPerPR": 0.0984,
      "costPerBugFound": 0.0686,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-08T01:39:11.584Z",
      "url": "https://codereviewbench.com/model/glm-5.3-flash"
    },
    {
      "id": "deepseek-v4-pro",
      "name": "DeepSeek V4 Pro",
      "provider": "DeepSeek",
      "mode": "balanced",
      "rank": 9,
      "f2": 34.5,
      "recall": 32.2,
      "precision": 47.6,
      "f1": 38.4,
      "bugsFound": 39,
      "bugsTotal": 121,
      "comments": 82,
      "pullRequests": 30,
      "costPerPR": 0.4898,
      "costPerBugFound": 0.3768,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T18:24:53.246Z",
      "url": "https://codereviewbench.com/model/deepseek-v4-pro"
    },
    {
      "id": "qwen3.8-27b",
      "name": "Qwen3.8 27B",
      "provider": "Alibaba",
      "mode": "balanced",
      "rank": 10,
      "f2": 30.9,
      "recall": 28.1,
      "precision": 51.5,
      "f1": 36.4,
      "bugsFound": 34,
      "bugsTotal": 121,
      "comments": 66,
      "pullRequests": 30,
      "costPerPR": 0.1984,
      "costPerBugFound": 0.1751,
      "costBasis": "api",
      "costNote": "Served by AkashML in fp8 (8-bit quantized weights).",
      "accessPath": "api",
      "runAt": "2026-11-09T16:45:26.145Z",
      "url": "https://codereviewbench.com/model/qwen3.8-27b"
    },
    {
      "id": "minimax-m3",
      "name": "MiniMax M3",
      "provider": "MiniMax",
      "mode": "balanced",
      "rank": 11,
      "f2": 21.9,
      "recall": 19.8,
      "precision": 38.1,
      "f1": 26.1,
      "bugsFound": 24,
      "bugsTotal": 121,
      "comments": 63,
      "pullRequests": 30,
      "costPerPR": 0.1543,
      "costPerBugFound": 0.1929,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-08T01:52:04.323Z",
      "url": "https://codereviewbench.com/model/minimax-m3"
    },
    {
      "id": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "provider": "Google",
      "mode": "balanced",
      "rank": 12,
      "f2": 18.2,
      "recall": 15.7,
      "precision": 50,
      "f1": 23.9,
      "bugsFound": 19,
      "bugsTotal": 121,
      "comments": 38,
      "pullRequests": 30,
      "costPerPR": 0.3485,
      "costPerBugFound": 0.5503,
      "costBasis": "api",
      "costNote": "Promotional price, valid until 2026-12-31.",
      "accessPath": "api",
      "runAt": "2026-11-07T15:41:56.276Z",
      "url": "https://codereviewbench.com/model/gemini-3.8-flash"
    },
    {
      "id": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "provider": "OpenAI",
      "mode": "balanced",
      "rank": 13,
      "f2": 8.1,
      "recall": 6.6,
      "precision": 61.5,
      "f1": 11.9,
      "bugsFound": 8,
      "bugsTotal": 121,
      "comments": 13,
      "pullRequests": 30,
      "costPerPR": 0.0229,
      "costPerBugFound": 0.0858,
      "costBasis": "api",
      "costNote": "Ran at the vendor's default reasoning effort, which reasons very little on this task (about 25 reasoning tokens per PR).",
      "accessPath": "api",
      "runAt": "2026-11-07T21:08:37.793Z",
      "url": "https://codereviewbench.com/model/gpt-6-luna"
    },
    {
      "id": "kimi-k3.1-pro",
      "name": "Kimi K3.1 Pro",
      "provider": "Moonshot",
      "mode": "heavy",
      "rank": 1,
      "f2": 58.4,
      "recall": 67.8,
      "precision": 37.6,
      "f1": 48.4,
      "bugsFound": 82,
      "bugsTotal": 121,
      "comments": 218,
      "pullRequests": 30,
      "costPerPR": 2.306,
      "costPerBugFound": 0.8436,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T21:37:28.908Z",
      "url": "https://codereviewbench.com/model/kimi-k3.1-pro"
    },
    {
      "id": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "provider": "OpenAI",
      "mode": "heavy",
      "rank": 2,
      "f2": 55.4,
      "recall": 55.4,
      "precision": 55.4,
      "f1": 55.4,
      "bugsFound": 67,
      "bugsTotal": 121,
      "comments": 121,
      "pullRequests": 30,
      "costPerPR": 1.7413,
      "costPerBugFound": 0.7797,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T20:57:45.785Z",
      "url": "https://codereviewbench.com/model/gpt-6.1-sol"
    },
    {
      "id": "glm-5.3",
      "name": "GLM-5.3",
      "provider": "Zhipu",
      "mode": "heavy",
      "rank": 3,
      "f2": 50.2,
      "recall": 55.4,
      "precision": 36.6,
      "f1": 44.1,
      "bugsFound": 67,
      "bugsTotal": 121,
      "comments": 183,
      "pullRequests": 30,
      "costPerPR": 2.5386,
      "costPerBugFound": 1.1367,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T23:49:55.548Z",
      "url": "https://codereviewbench.com/model/glm-5.3"
    },
    {
      "id": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "provider": "Anthropic",
      "mode": "heavy",
      "rank": 4,
      "f2": 49.4,
      "recall": 49.6,
      "precision": 48.8,
      "f1": 49.2,
      "bugsFound": 60,
      "bugsTotal": 121,
      "comments": 123,
      "pullRequests": 30,
      "costPerPR": 3.0999,
      "costPerBugFound": 1.5499,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-09T13:40:57.393Z",
      "url": "https://codereviewbench.com/model/claude-opus-5.5"
    },
    {
      "id": "deepseek-v4.1-flash",
      "name": "DeepSeek V4.1 Flash",
      "provider": "DeepSeek",
      "mode": "heavy",
      "rank": 5,
      "f2": 48.9,
      "recall": 52.9,
      "precision": 37.6,
      "f1": 44,
      "bugsFound": 64,
      "bugsTotal": 121,
      "comments": 170,
      "pullRequests": 30,
      "costPerPR": 0.3405,
      "costPerBugFound": 0.1596,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T22:01:17.894Z",
      "url": "https://codereviewbench.com/model/deepseek-v4.1-flash"
    },
    {
      "id": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "provider": "Anthropic",
      "mode": "heavy",
      "rank": 6,
      "f2": 48.8,
      "recall": 47.1,
      "precision": 57,
      "f1": 51.6,
      "bugsFound": 57,
      "bugsTotal": 121,
      "comments": 100,
      "pullRequests": 30,
      "costPerPR": 1.7544,
      "costPerBugFound": 0.9234,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T19:12:04.585Z",
      "url": "https://codereviewbench.com/model/claude-sonnet-5.5"
    },
    {
      "id": "deepseek-v4-pro",
      "name": "DeepSeek V4 Pro",
      "provider": "DeepSeek",
      "mode": "heavy",
      "rank": 7,
      "f2": 46.4,
      "recall": 47.9,
      "precision": 41.1,
      "f1": 44.3,
      "bugsFound": 58,
      "bugsTotal": 121,
      "comments": 141,
      "pullRequests": 30,
      "costPerPR": 1.2959,
      "costPerBugFound": 0.6703,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T22:08:01.032Z",
      "url": "https://codereviewbench.com/model/deepseek-v4-pro"
    },
    {
      "id": "glm-5.3-flash",
      "name": "GLM-5.3 Flash",
      "provider": "Zhipu",
      "mode": "heavy",
      "rank": 8,
      "f2": 46.3,
      "recall": 47.9,
      "precision": 40.6,
      "f1": 43.9,
      "bugsFound": 58,
      "bugsTotal": 121,
      "comments": 143,
      "pullRequests": 30,
      "costPerPR": 0.2207,
      "costPerBugFound": 0.1141,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-08T03:40:09.874Z",
      "url": "https://codereviewbench.com/model/glm-5.3-flash"
    },
    {
      "id": "qwen3.8-27b",
      "name": "Qwen3.8 27B",
      "provider": "Alibaba",
      "mode": "heavy",
      "rank": 9,
      "f2": 42.3,
      "recall": 43,
      "precision": 39.7,
      "f1": 41.3,
      "bugsFound": 52,
      "bugsTotal": 121,
      "comments": 131,
      "pullRequests": 30,
      "costPerPR": 0.409,
      "costPerBugFound": 0.236,
      "costBasis": "api",
      "costNote": "Served by AkashML in fp8 (8-bit quantized weights).",
      "accessPath": "api",
      "runAt": "2026-11-09T19:58:45.545Z",
      "url": "https://codereviewbench.com/model/qwen3.8-27b"
    },
    {
      "id": "muse-spark-1.2",
      "name": "Muse Spark 1.2",
      "provider": "Meta",
      "mode": "heavy",
      "rank": 10,
      "f2": 40.6,
      "recall": 46.3,
      "precision": 27.3,
      "f1": 34.4,
      "bugsFound": 56,
      "bugsTotal": 121,
      "comments": 205,
      "pullRequests": 30,
      "costPerPR": 2.0724,
      "costPerBugFound": 1.1102,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-07T19:41:57.563Z",
      "url": "https://codereviewbench.com/model/muse-spark-1.2"
    },
    {
      "id": "minimax-m3",
      "name": "MiniMax M3",
      "provider": "MiniMax",
      "mode": "heavy",
      "rank": 11,
      "f2": 29,
      "recall": 28.1,
      "precision": 33.3,
      "f1": 30.5,
      "bugsFound": 34,
      "bugsTotal": 121,
      "comments": 102,
      "pullRequests": 30,
      "costPerPR": 0.3179,
      "costPerBugFound": 0.2805,
      "costBasis": "api",
      "costNote": null,
      "accessPath": "api",
      "runAt": "2026-11-08T04:24:01.523Z",
      "url": "https://codereviewbench.com/model/minimax-m3"
    },
    {
      "id": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "provider": "Google",
      "mode": "heavy",
      "rank": 12,
      "f2": 24.2,
      "recall": 21.5,
      "precision": 48.1,
      "f1": 29.7,
      "bugsFound": 26,
      "bugsTotal": 121,
      "comments": 54,
      "pullRequests": 30,
      "costPerPR": 0.7379,
      "costPerBugFound": 0.8514,
      "costBasis": "api",
      "costNote": "Promotional price, valid until 2026-12-31.",
      "accessPath": "api",
      "runAt": "2026-11-07T16:45:02.577Z",
      "url": "https://codereviewbench.com/model/gemini-3.8-flash"
    },
    {
      "id": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "provider": "OpenAI",
      "mode": "heavy",
      "rank": 13,
      "f2": 15.1,
      "recall": 13.2,
      "precision": 35.6,
      "f1": 19.3,
      "bugsFound": 16,
      "bugsTotal": 121,
      "comments": 45,
      "pullRequests": 30,
      "costPerPR": 0.0571,
      "costPerBugFound": 0.1071,
      "costBasis": "api",
      "costNote": "Ran at the vendor's default reasoning effort, which reasons very little on this task (about 25 reasoning tokens per PR).",
      "accessPath": "api",
      "runAt": "2026-11-07T21:33:05.798Z",
      "url": "https://codereviewbench.com/model/gpt-6-luna"
    }
  ]
}