{"datasets":[{"id":"ds_coding_golden","name":"Coding golden set","description":"Golden coding tasks: implementation, refactoring, debugging and review prompts with expected topical coverage.","taskFamily":"coding","itemCount":8,"items":[{"id":"coding-01","prompt":"Write a TypeScript debounce utility with cancel support and explain the closure lifecycle around the timer handle.","expected":{"keywords":["debounce","timer","cancel"],"minChars":200,"maxChars":6000}},{"id":"coding-02","prompt":"Refactor a nested callback pyramid into async await style and describe the error propagation strategy you chose.","expected":{"keywords":["async","await","error"],"minChars":200,"maxChars":6000}},{"id":"coding-03","prompt":"Diagnose a memory leak caused by an unremoved event listener inside a React component and outline the cleanup fix.","expected":{"keywords":["memory","listener","cleanup"],"minChars":200,"maxChars":6000}},{"id":"coding-04","prompt":"Design a retry helper with exponential backoff and jitter for flaky network requests, covering the maximum attempt budget.","expected":{"keywords":["retry","backoff","jitter"],"minChars":200,"maxChars":6000}},{"id":"coding-05","prompt":"Review a SQL migration that adds a unique index to a large table and explain how to avoid locking production writes.","expected":{"keywords":["migration","index","locking"],"minChars":200,"maxChars":6000}},{"id":"coding-06","prompt":"Implement an LRU cache class with constant time reads and describe the eviction order guarantees your structure provides.","expected":{"keywords":["cache","eviction","constant"],"minChars":200,"maxChars":6000}},{"id":"coding-07","prompt":"Explain how to paginate a REST endpoint with stable cursor tokens and why offset pagination breaks under concurrent inserts.","expected":{"keywords":["cursor","pagination","offset"],"minChars":200,"maxChars":6000}},{"id":"coding-08","prompt":"Harden a file upload handler against oversized payloads and unsafe filename traversal, listing each validation step.","expected":{"keywords":["upload","validation","traversal"],"minChars":200,"maxChars":6000}}]},{"id":"ds_summarization_golden","name":"Summarization golden set","description":"Golden summarization tasks: incident notes, meeting minutes and policy digests with concise-answer length windows.","taskFamily":"summarization","itemCount":7,"items":[{"id":"summarization-01","prompt":"Summarize an incident report: the payment gateway returned elevated latency for forty minutes after a certificate rotation, mitigated by rollback.","expected":{"keywords":["latency","certificate","rollback"],"minChars":120,"maxChars":2500}},{"id":"summarization-02","prompt":"Summarize meeting minutes where the platform team agreed to migrate batch workloads to committed capacity before the contract renewal deadline.","expected":{"keywords":["batch","capacity","renewal"],"minChars":120,"maxChars":2500}},{"id":"summarization-03","prompt":"Summarize a security bulletin describing a credential stuffing campaign against the partner portal and the mandatory rotation response.","expected":{"keywords":["credential","portal","rotation"],"minChars":120,"maxChars":2500}},{"id":"summarization-04","prompt":"Summarize a quarterly finance memo: cloud spend grew twelve percent while inference unit economics improved through caching.","expected":{"keywords":["spend","economics","caching"],"minChars":120,"maxChars":2500}},{"id":"summarization-05","prompt":"Summarize a product update thread announcing the deprecation timeline for the legacy embeddings endpoint and the replacement guidance.","expected":{"keywords":["deprecation","embeddings","guidance"],"minChars":120,"maxChars":2500}},{"id":"summarization-06","prompt":"Summarize an oncall handoff covering two open alerts, one silenced flapping monitor and the pending capacity review.","expected":{"keywords":["alerts","monitor","capacity"],"minChars":120,"maxChars":2500}},{"id":"summarization-07","prompt":"Summarize a compliance review noting the audit trail gaps in the reporting pipeline and the remediation owners assigned.","expected":{"keywords":["audit","pipeline","remediation"],"minChars":120,"maxChars":2500}}]},{"id":"ds_extraction_json","name":"Extraction JSON set","description":"Structured extraction tasks graded primarily on JSON validity plus field-name coverage.","taskFamily":"extraction","itemCount":6,"items":[{"id":"extraction-01","prompt":"Extract vendor, amount and currency from this invoice line as JSON: 'Acme Networks — 1,240.00 USD due on receipt'.","expected":{"mustBeJson":true,"keywords":["vendor","amount","currency"],"minChars":20,"maxChars":2000}},{"id":"extraction-02","prompt":"Extract name, email and company from this signature as JSON: 'Mei Lin, Head of Data, mei.lin@harborlight.io, Harborlight'.","expected":{"mustBeJson":true,"keywords":["name","email","company"],"minChars":20,"maxChars":2000}},{"id":"extraction-03","prompt":"Extract origin, destination and departure date from this itinerary as JSON: 'SIN to KUL, departing 2026-08-14 morning'.","expected":{"mustBeJson":true,"keywords":["origin","destination","departure"],"minChars":20,"maxChars":2000}},{"id":"extraction-04","prompt":"Extract severity, service and region from this alert as JSON: 'P2 elevated errors on checkout service in ap-southeast-1'.","expected":{"mustBeJson":true,"keywords":["severity","service","region"],"minChars":20,"maxChars":2000}},{"id":"extraction-05","prompt":"Extract title, quantity and unit price from this order row as JSON: '3x ergonomic keyboard at 89.50 each'.","expected":{"mustBeJson":true,"keywords":["title","quantity","price"],"minChars":20,"maxChars":2000}},{"id":"extraction-06","prompt":"Extract policy number, holder and expiry from this notice as JSON: 'Policy MY-88231 held by Tan Wei expires 2027-01-31'.","expected":{"mustBeJson":true,"keywords":["policy","holder","expiry"],"minChars":20,"maxChars":2000}}]}],"runs":[{"id":"run_1588ba8odt","datasetId":"ds_extraction_json","modelIds":["openai/gpt-5.2","google/gemini-3-pro","anthropic/claude-sonnet"],"status":"completed","results":[{"itemId":"extraction-01","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1263,"costUsd":0.0045},{"itemId":"extraction-02","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1587,"costUsd":0.0044},{"itemId":"extraction-03","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2592,"costUsd":0.0017},{"itemId":"extraction-04","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2537,"costUsd":0.0044},{"itemId":"extraction-05","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2402,"costUsd":0.0032},{"itemId":"extraction-06","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1376,"costUsd":0.0012},{"itemId":"extraction-01","modelId":"google/gemini-3-pro","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2255,"costUsd":0.0016},{"itemId":"extraction-02","modelId":"google/gemini-3-pro","generationId":null,"score":0.8833,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2255,"costUsd":0.003},{"itemId":"extraction-03","modelId":"google/gemini-3-pro","generationId":null,"score":0.8833,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":1918,"costUsd":0.0011},{"itemId":"extraction-04","modelId":"google/gemini-3-pro","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2209,"costUsd":0.0032},{"itemId":"extraction-05","modelId":"google/gemini-3-pro","generationId":null,"score":0.8833,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2461,"costUsd":0.0039},{"itemId":"extraction-06","modelId":"google/gemini-3-pro","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2316,"costUsd":0.006},{"itemId":"extraction-01","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1258,"costUsd":0.0063},{"itemId":"extraction-02","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1446,"costUsd":0.0018},{"itemId":"extraction-03","modelId":"anthropic/claude-sonnet","generationId":null,"score":0.8833,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2001,"costUsd":0.0045},{"itemId":"extraction-04","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2550,"costUsd":0.0066},{"itemId":"extraction-05","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":951,"costUsd":0.0017},{"itemId":"extraction-06","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1446,"costUsd":0.0035}],"scorecardIds":["sc_rmkpr2li0k","sc_aju7gbgkhy","sc_l2szxudubg"],"triggeredBy":"seed","startedAt":1785189624435,"completedAt":1785189714435,"rolloutId":"ro_4ygphfac0l"},{"id":"run_gnmqri1dvt","datasetId":"ds_coding_golden","modelIds":["anthropic/claude-sonnet","openai/gpt-5.2","anthropic/claude-haiku"],"status":"completed","results":[{"itemId":"coding-01","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2560,"costUsd":0.0033},{"itemId":"coding-02","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1962,"costUsd":0.0026},{"itemId":"coding-03","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2252,"costUsd":0.0049},{"itemId":"coding-04","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2430,"costUsd":0.0022},{"itemId":"coding-05","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2450,"costUsd":0.0031},{"itemId":"coding-06","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2201,"costUsd":0.006},{"itemId":"coding-07","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2064,"costUsd":0.0047},{"itemId":"coding-08","modelId":"anthropic/claude-sonnet","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2021,"costUsd":0.0023},{"itemId":"coding-01","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2429,"costUsd":0.0018},{"itemId":"coding-02","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":788,"costUsd":0.0054},{"itemId":"coding-03","modelId":"openai/gpt-5.2","generationId":null,"score":0.7834,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2568,"costUsd":0.004},{"itemId":"coding-04","modelId":"openai/gpt-5.2","generationId":null,"score":0.7834,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":1097,"costUsd":0.0042},{"itemId":"coding-05","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1324,"costUsd":0.0064},{"itemId":"coding-06","modelId":"openai/gpt-5.2","generationId":null,"score":0.7834,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2532,"costUsd":0.0065},{"itemId":"coding-07","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1705,"costUsd":0.0032},{"itemId":"coding-08","modelId":"openai/gpt-5.2","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1412,"costUsd":0.0009},{"itemId":"coding-01","modelId":"anthropic/claude-haiku","generationId":null,"score":0.7834,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":983,"costUsd":0.0031},{"itemId":"coding-02","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1635,"costUsd":0.0032},{"itemId":"coding-03","modelId":"anthropic/claude-haiku","generationId":null,"score":0.7834,"components":{"jsonValidity":1,"keywordCoverage":0.6667,"lengthSanity":1},"latencyMs":2482,"costUsd":0.0053},{"itemId":"coding-04","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2264,"costUsd":0.002},{"itemId":"coding-05","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1045,"costUsd":0.0045},{"itemId":"coding-06","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2320,"costUsd":0.0033},{"itemId":"coding-07","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":2128,"costUsd":0.006},{"itemId":"coding-08","modelId":"anthropic/claude-haiku","generationId":null,"score":1,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"latencyMs":1476,"costUsd":0.0024}],"scorecardIds":["sc_f78giz0ah0","sc_1icupoo9oa","sc_kfo8y3rt88"],"triggeredBy":"seed","startedAt":1784584824435,"completedAt":1784584914435}],"scorecards":[{"id":"sc_rmkpr2li0k","runId":"run_1588ba8odt","datasetId":"ds_extraction_json","modelId":"openai/gpt-5.2","taskFamily":"extraction","score":1,"passRate":1,"itemCount":6,"meanLatencyMs":1960,"totalCostUsd":0.0194,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"createdAt":1785189714435},{"id":"sc_aju7gbgkhy","runId":"run_1588ba8odt","datasetId":"ds_extraction_json","modelId":"google/gemini-3-pro","taskFamily":"extraction","score":0.9417,"passRate":1,"itemCount":6,"meanLatencyMs":2236,"totalCostUsd":0.0188,"components":{"jsonValidity":1,"keywordCoverage":0.8334,"lengthSanity":1},"createdAt":1785189714435},{"id":"sc_l2szxudubg","runId":"run_1588ba8odt","datasetId":"ds_extraction_json","modelId":"anthropic/claude-sonnet","taskFamily":"extraction","score":0.9806,"passRate":1,"itemCount":6,"meanLatencyMs":1609,"totalCostUsd":0.0244,"components":{"jsonValidity":1,"keywordCoverage":0.9445,"lengthSanity":1},"createdAt":1785189714435},{"id":"sc_f78giz0ah0","runId":"run_gnmqri1dvt","datasetId":"ds_coding_golden","modelId":"anthropic/claude-sonnet","taskFamily":"coding","score":1,"passRate":1,"itemCount":8,"meanLatencyMs":2243,"totalCostUsd":0.0291,"components":{"jsonValidity":1,"keywordCoverage":1,"lengthSanity":1},"createdAt":1784584914435},{"id":"sc_1icupoo9oa","runId":"run_gnmqri1dvt","datasetId":"ds_coding_golden","modelId":"openai/gpt-5.2","taskFamily":"coding","score":0.9188,"passRate":1,"itemCount":8,"meanLatencyMs":1732,"totalCostUsd":0.0324,"components":{"jsonValidity":1,"keywordCoverage":0.875,"lengthSanity":1},"createdAt":1784584914435},{"id":"sc_kfo8y3rt88","runId":"run_gnmqri1dvt","datasetId":"ds_coding_golden","modelId":"anthropic/claude-haiku","taskFamily":"coding","score":0.9459,"passRate":1,"itemCount":8,"meanLatencyMs":1792,"totalCostUsd":0.0298,"components":{"jsonValidity":1,"keywordCoverage":0.9167,"lengthSanity":1},"createdAt":1784584914435}],"routerVersions":[{"id":"rv_bgqhrj3u4y","version":"router_1.0+eval.run_1588ba8odt","baseRouterVersion":"router_1.0","runId":"run_1588ba8odt","proposedQualityScores":{"openai/gpt-5.2":{"extraction":0.944},"google/gemini-3-pro":{"extraction":0.9125},"anthropic/claude-sonnet":{"extraction":0.9382}},"createdAt":1785189714435}],"rollouts":[{"id":"ro_4ygphfac0l","routerVersionId":"rv_bgqhrj3u4y","runId":"run_1588ba8odt","stage":"canary","history":[{"stage":"shadow","at":1785189714435},{"stage":"canary","at":1785276114435}],"applied":false,"createdAt":1785189714435,"updatedAt":1785276114435}],"feedback":[{"id":"fb_ahwhwqjcg6","generationId":"gen_bxa9cnizle","principalId":"prin_admin","rating":1,"createdAt":1785352450502,"comment":"Fast and grounded; keep routing these to this model."},{"id":"fb_3wvwbas7re","generationId":"gen_u1l3vrtgkk","principalId":"prin_agent","rating":1,"createdAt":1785275620359},{"id":"fb_k5c0620nhr","generationId":"gen_xa3lbqedvm","principalId":"prin_app","rating":-1,"createdAt":1785184785430,"comment":"Answer ignored the JSON schema we asked for."},{"id":"fb_a46ipm61xq","generationId":"gen_9z984r0d2g","principalId":"prin_admin","rating":1,"createdAt":1785079858925,"comment":"Accurate and well structured — exactly what the runbook needed."}]}