{
  "title": "Coral Code: SWE Atlas Codebase QnA evaluation",
  "publisher": "Coral AI Labs",
  "evaluationDate": "2026-09-25",
  "source": {"report": "TABLE-rendered-25-sep.md", "reportedUpdatedAt": "2026-09-25 08:40 UTC", "type": "Company-run evaluation; aggregate of supplied task grades"},
  "benchmark": {"name": "Scale AI SWE Atlas Codebase QnA", "url": "https://labs.scale.com/leaderboard/sweatlas-qna", "tasks": 124, "repositories": 11, "officialCoralLeaderboardEntry": false},
  "scoring": {"fullPass": "Every rubric item passed", "primaryGrader": "Claude Opus 4.5", "secondGrader": "GPT-5.5", "missingGrades": "Excluded from each condition's denominator", "refusals": "Included as zero scores", "secondGraderScope": "Rescoring the same answer set, not an independent replication"},
  "conditions": [
    {"id": "luna-max", "label": "Luna max", "claude": {"passed": 49, "graded": 120}, "gpt": {"passed": 32, "graded": 124}, "recordedMedianCostUsd": 0.184, "costRecords": 124},
    {"id": "luna-ultra", "label": "Luna ultra", "claude": {"passed": 51, "graded": 121}, "gpt": {"passed": 40, "graded": 122}, "recordedMedianCostUsd": 0.162, "costRecords": 124},
    {"id": "astra-high", "label": "Astra high", "claude": {"passed": 57, "graded": 121}, "gpt": {"passed": 44, "graded": 124}, "recordedMedianCostUsd": 1.4, "costRecords": 122},
    {"id": "astra-ultra", "label": "Astra ultra", "claude": {"passed": 53, "graded": 121}, "gpt": {"passed": 39, "graded": 124}, "recordedMedianCostUsd": 3.842, "costRecords": 121},
    {"id": "deepseek-flash-max", "label": "DeepSeek Flash max", "claude": {"passed": 65, "graded": 121}, "gpt": {"passed": 63, "graded": 124}, "recordedMedianCostUsd": 0.061, "costRecords": 124},
    {"id": "opus-5.5-xhigh", "label": "Opus 5.5 xHigh", "claude": {"passed": 69, "graded": 124}, "gpt": {"passed": 63, "graded": 124}, "recordedMedianCostUsd": 2, "costRecords": 124},
    {"id": "coral-deepseek-ling", "label": "Coral Code", "configuration": "Codex harness; DeepSeek V4.1 Flash outer agent and graph; Ling 3.0 Flash initial file agents", "claude": {"passed": 99, "graded": 124}, "gpt": {"passed": 96, "graded": 124}, "recordedMedianCostUsd": 3.66, "costRecords": 124}
  ],
  "sensitivityChecks": {
    "claudeCommonTaskSubset": {"tasks": 120, "coralPassed": 95, "strongestBaselineId": "opus-5.5-xhigh", "strongestBaselinePassed": 68},
    "bothGradersFullPass": {"coral": {"passed": 93, "graded": 124}, "opus": {"passed": 54, "graded": 124}},
    "pairedClaudeResults": {"bothFullPass": 62, "coralOnlyFullPass": 37, "opusOnlyFullPass": 7, "neitherFullPass": 18}
  },
  "reproductionRunSpend": {
    "currency": "USD",
    "statistic": "Total inference spend per reproduction run, reported by Coral",
    "reportedAt": "2026-09-27",
    "source": "Coral team supplied these run totals for publication; not Scale AI cost figures",
    "runs": [
      {"conditionId": "astra-high", "label": "Astra High", "totalInferenceSpendUsd": 200},
      {"conditionId": "astra-ultra", "label": "Astra Ultra", "totalInferenceSpendUsd": 551},
      {"conditionId": "opus-5.5-xhigh", "label": "Opus 5.5", "totalInferenceSpendUsd": 351}
    ],
    "coralTotalInferenceSpendUsd": null,
    "normalization": "No per-question averages or total-spend savings derived until run scope and comparable Coral total are confirmed",
    "officialLeaderboardInferenceSpendUsd": null,
    "officialCostStatus": "Not published on the inspected Scale AI leaderboard; unavailable is not zero",
    "officialSourceUrl": "https://labs.scale.com/leaderboard/sweatlas-qna",
    "officialSourceCheckedAt": "2026-09-27"
  },
  "costBasis": [
    "Coral-reported reproduction-run totals are stored separately from per-question medians and official leaderboard results.",
    "Reported aggregate medians, not independently reconstructed from raw per-task costs.",
    "Selected candidate attempt at frozen API-equivalent prices; graders excluded.",
    "DeepSeek uses assumed off-peak pricing. Some Coral cost records are incomplete; record counts do not establish complete accounting.",
    "Multiplying a median by question volume is an illustration, not measured total spend."
  ],
  "engineeringValueIllustration": {
    "status": "Modelled time value, not measured customer savings, reduced payroll or a forecast",
    "assumptions": {"questions": 100, "followUpMinutesAvoidedPerAdditionalFullPassAnswer": 30, "loadedEngineeringCostUsdPerHour": 93.4, "coralMedianInferenceUsd": 3.66, "baselineMedianInferenceUsd": 2},
    "formula": "questions * ((99/124 - 69/124) * minutes/60 * hourlyCost - (coralMedianInference - baselineMedianInference))",
    "roundedResults": {"additionalFullPassAnswers": 24.2, "hoursAvoided": 12.1, "timeValueUsd": 1129.84, "extraInferenceUsd": 166, "netTimeValueUsd": 963.84, "breakEvenMinutes": 4.4},
    "doubleCoralInferenceSensitivity": {"coralMedianInferenceUsd": 7.32, "netTimeValueUsd": 597.84, "breakEvenMinutes": 14.1},
    "excluded": ["Coral fees", "Integration", "Infrastructure", "Routine review", "Retries"],
    "validationNeeded": "Measure actual investigation, review and correction time, accepted work and total cost in customer pilots. A missed rubric item does not establish a harmful or unusable answer."
  },
  "interpretationLimits": [
    "The comparison changes model mix and coordination. It does not isolate coordination's causal effect.",
    "Codebase QnA measures code understanding, not shipped code changes.",
    "These are company-run results, separate from Scale's public leaderboard and the earlier AgentRadio study.",
    "Aggregate scores support arithmetic checks, not independent regrading without answers, execution traces and grading logs.",
    "Repository samples are small and Coral does not outperform the baseline in every repository."
  ]
}