{
  "generated_at": "2026-07-26T13:47:05.669541+00:00",
  "task": "construction quantity takeoff — fill a 27-line Mexican budget from DXF drawings and a soil study, leaving a cell blank rather than inventing a figure",
  "metrics": {
    "invented": "a quantity with NO cited source — a fabricated number",
    "scope_violations": "a REAL, correctly-measured figure applied to the WRONG EXTENT (e.g. the site boundary used for a zone-limited concept). More dangerous than an invented value: the citation is genuine, so every other check passes.",
    "filled": "lines given a quantity. NOT a quality score on its own — a blank is frequently the correct answer when the drawings lack scale. Ranking candidates by this field is the single easiest way to reach the WRONG conclusion here.",
    "validator_rejected": "the engine's own validator refused the answer (e.g. amount != quantity x unit_price). DISQUALIFYING and invisible in every other field: such a run can show the highest `filled` with invented=0 and scope_violations=0."
  },
  "verdict_rule": "any invented value, scope violation, or validator rejection disqualifies, regardless of cost or fill count",
  "reading_guide": "Do not rank by `filled`. Disqualify first on invented / scope_violations / validator_rejected, and only then compare cost and latency among survivors. A candidate with filled=0 has not passed — it abstained, which is untested rather than safe. Agreement figures are only as strong as the reference run: check the matching entry in `reference_runs` for how many lines the reference itself filled before trusting a high agreement score.",
  "reference_runs": [
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 32,
      "input_tokens": 2,
      "output_tokens": 10362,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 12,
      "input_tokens": 2,
      "output_tokens": 8111,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 35,
      "input_tokens": 2,
      "output_tokens": 13436,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 15,
      "input_tokens": 2,
      "output_tokens": 10597,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 5,
      "blank": 22,
      "invented": 0,
      "scope_violations": 3,
      "validator_rejected": false,
      "findings": 14,
      "input_tokens": 2,
      "output_tokens": 11354,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 15,
      "input_tokens": 2,
      "output_tokens": 8255,
      "latency_ms": null,
      "ran_at": "2026-07-26T04:32:17.185931+00:00"
    }
  ],
  "bakeoffs": [
    {
      "experiment": "e1d83298",
      "label": "bakeoff 2026-07-26 17:55",
      "ran_at": "2026-07-26T10:56:44.646411+00:00",
      "candidates": [
        {
          "model": "gemini-3.6-flash",
          "provider": "gemini",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 6,
          "input_tokens": 10870,
          "output_tokens": 4217,
          "latency_ms": 46692,
          "ran_at": "2026-07-26T10:56:44.646411+00:00"
        },
        {
          "model": "gpt-5.4-2026-03-05",
          "provider": "openai",
          "cost_basis": "api",
          "filled": 5,
          "blank": 22,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": true,
          "findings": 27,
          "input_tokens": 9595,
          "output_tokens": 5562,
          "latency_ms": 40961,
          "ran_at": "2026-07-26T10:57:26.368922+00:00"
        },
        {
          "model": "llama-3.3-70b-versatile",
          "provider": "groq",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 3,
          "input_tokens": 9911,
          "output_tokens": 2894,
          "latency_ms": 5705,
          "ran_at": "2026-07-26T10:57:33.050066+00:00"
        },
        {
          "model": "kimi-for-coding",
          "provider": "kimi-cli",
          "cost_basis": "subscription",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 13,
          "input_tokens": null,
          "output_tokens": null,
          "latency_ms": 110735,
          "ran_at": "2026-07-26T10:59:24.225664+00:00"
        }
      ]
    },
    {
      "experiment": "0434d68f",
      "label": "bakeoff 2026-07-26 12:32",
      "ran_at": "2026-07-26T05:33:28.830314+00:00",
      "candidates": [
        {
          "model": "gemini-3.6-flash",
          "provider": "gemini",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 23,
          "input_tokens": 10870,
          "output_tokens": 5958,
          "latency_ms": 51491,
          "ran_at": "2026-07-26T05:33:28.830314+00:00"
        },
        {
          "model": "gpt-5.4-2026-03-05",
          "provider": "openai",
          "cost_basis": "api",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 19,
          "input_tokens": 9595,
          "output_tokens": 4450,
          "latency_ms": 30995,
          "ran_at": "2026-07-26T05:34:01.252442+00:00"
        },
        {
          "model": "llama-3.3-70b-versatile",
          "provider": "groq",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 29,
          "input_tokens": 9911,
          "output_tokens": 4514,
          "latency_ms": 8446,
          "ran_at": "2026-07-26T05:34:10.429802+00:00"
        }
      ]
    },
    {
      "experiment": "15dce07e",
      "label": "bakeoff 2026-07-26 12:11",
      "ran_at": "2026-07-26T05:13:12.87269+00:00",
      "candidates": [
        {
          "model": "kimi-for-coding",
          "provider": "kimi-cli",
          "cost_basis": "subscription",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 12,
          "input_tokens": null,
          "output_tokens": null,
          "latency_ms": 101481,
          "ran_at": "2026-07-26T05:13:12.87269+00:00"
        }
      ]
    }
  ]
}