{"schemaVersion":1,"snapshotRetrievedAt":"2026-10-09T03:19:52Z","title":"ARC public-evaluation matched-task snapshot","status":"validated_pinned_sources_and_derived_statistics","license":{"spdx":"MIT","basis":"Both official ARC Prize Hugging Face dataset cards declare MIT. Preserve attribution and MIT notice; this snapshot does not copy ARC website leaderboard assets.","sources":["https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval","https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval"]},"methodology":{"scope":"Abstract grid reasoning only. Two public benchmark versions; no coding, general intelligence or production-quality claim.","scores":"For each benchmark version, retain the task-ID intersection across all five configurations. Score = 100 × arithmetic mean of the publisher task_results[task_id].score. This is a SIENG-derived matched-subset score, NOT the official ARC semi-private leaderboard score.","cost":"USD per task = arithmetic mean of publisher task_results[task_id].cost on the exact same matched IDs. Historical published run costs, based on retail API rates at run time; not current pricing or a fresh measurement.","missing":"Excluded tasks are missing in at least one source run. Common-case filtering can introduce survivorship bias; coverage is shown. Do not fill missing values with zero or mix unmatched denominators.","attempts":"Publisher per-task credit: each test pair receives credit if either attempt is exactly correct; task score averages its pairs. Sum of all recorded attempt costs. Different reasoning configurations retain their exact labels.","latency":"Not exposed. Publisher duration is summed request elapsed time across attempts, not wall-clock task latency or TTFT.","combinations":"Do not average two model scores to claim ensemble performance. Routed scenario estimates must choose one published configuration per task class and be labelled derived. No measured ensemble or live API runs.","sourceScorer":"https://github.com/arcprize/arc-agi-benchmarking/blob/main/src/arc_agi_benchmarking/scoring/scoring.py","configurations":"Same matched task IDs and publisher scoring. Token caps/reasoning settings differ by named configuration; this compares configured systems, not controlled equal-compute model weights. Observed request settings are stored per record from one linked sample task. They have not been exhaustively checked across all requests.","freshness":"Official published public-run archive last updated 2026-06-04, retrieved 2026-10-09. Selected sample requests range December 2025–May 2026. This is an archival snapshot, not a current frontier ranking.","uncertainty":"One published run per selected configuration; frontier is descriptive of this subset. No statistical superiority claim, confidence intervals, or robust production ranking."},"models":[{"id":"gpt-5-2-2025-12-11-thinking-high","label":"GPT-5.2 · high","provider":"openai","apiModelId":"gpt-5.2-2025-12-11","reasoningEffort":"high"},{"id":"claude-opus-4-8-high","label":"Claude Opus 4.8 · high","provider":"anthropic","apiModelId":"claude-opus-4-8","reasoningEffort":"high"},{"id":"claude-opus-4-6-thinking-120K-high","label":"Claude Opus 4.6 · high, 120K cap","provider":"anthropic","apiModelId":"claude-opus-4-6","reasoningEffort":"high"},{"id":"gemini-3-flash-preview-thinking-high","label":"Gemini 3 Flash Preview · high","provider":"google","apiModelId":"gemini-3-flash-preview","reasoningEffort":"high"},{"id":"gemini-3-1-pro-preview","label":"Gemini 3.1 Pro Preview","provider":"google","apiModelId":"gemini-3.1-pro-preview","reasoningEffort":null}],"benchmarks":[{"id":"arc-agi-1-public-matched","label":"ARC-AGI-1 · matched public subset","taskClass":"Abstract grid reasoning","matchedTasks":328,"fullPublicTasks":400,"coveragePct":82,"taskIds":["00576224","009d5c81","00dbd492","03560426","0607ce86","070dd51e","08573cc6","09c534e7","0a1d4ef5","0a2355a6","0b17323b","0bb8deee","0becf7df","0c786b71","0c9aba6e","0d87d2a6","0e671a1a","0f63c0b9","11e1fe23","12422b43","12997ef3","12eac192","136b0064","13713586","137f0df0","140c817e","14754a24","15113be4","15663ba9","15696249","16b78196","17b80ad2","17cae0c1","18419cfa","184a9768","195ba7dc","1a2e2828","1a6449f1","1acc24af","1c02dbbe","1c0d0a4b","1c56ad9f","1d0a4b61","1e81d6f9","2072aba6","20818e16","20981f0e","212895b5","21f83797","25094a63","2546ccf6","256b0a75","2685904e","27a77e38","27f8ce4f","292dd178","2a5f8217","2b01abd0","2c0b0aff","2c737e39","310f3251","3194b014","319f2597","31adaf00","31d5ba1a","32e9702f","332efdb3","3391f8c0","33b52de3","34b99a2b","351d6448","358ba94e","37d3e8b2","3979b1a8","3a301edc","3b4c2228","3d31c5b3","3ed85e70","3ee1011a","3f23242b","40f6cd08","414297c0","423a55dc","42918530","42a15761","4364c1c4","45737921","45bbe264","477d2879","48131b3c","4852f2fa","48f8583b","4aab4007","4acc7107","4c177718","4cd1b7b2","4e469f39","4f537728","505fff84","506d28a5","50a16a69","50aad11f","516b51b7","5289ad53","52fd389e","55059096","551d5bf1","55783887","5783df64","5833af48","58743b76","58e15b12","59341089","5a5a2103","5af49b42","5b526a93","5b692c0f","5b6cbef5","5d2a5c43","5ffb2104","604001fa","60a26a3e","626c0bcc","62b74c02","639f5a19","642248e4","642d658d","64a7c07e","66e6c45b","66f2d22f","67636eac","67b4a34d","67c52801","692cd3b6","695367ec","696d4842","69889d6e","6a11f6da","6ad5bdfd","6df30ad6","6ea4a07e","6f473927","7039b2d7","705a3229","712bf12e","72207abc","72a961c9","73182012","73c3b0d8","73ccf9c2","759f3fd3","762cd429","770cc55f","782b5218","79369cc6","7953d61e","7bb29440","7c8af763","7c9b52a0","7d18a6fb","7d1f7ee8","7e02026e","7ee1c6ea","817e6c09","81c0276b","833dafe3","845d6e51","8597cfd7","85b81ff1","85fa5666","8719f442","891232d6","8b28cd80","8ba14f53","8cb8642d","8dae5dfc","8e2edd66","8ee62060","8fbca751","90347967","903d1b4a","9110e3c5","917bccba","929ab4e9","92e50de0","9356391f","93b4f4b3","93c31fbe","94133066","94414823","94be5b80","95a58926","963f59bc","96a8c0cd","97239e3d","9772c176","981571dc","992798f6","99306f82","9a4bb226","9b2a60aa","9b365c51","9b4c17c4","9bebae7a","9c1e755f","9caba7c3","9def23fe","9f27f097","a04b2602","a096bf4d","a406ac07","a57f2f04","a8610ef7","aa18de87","aa300dc3","aab50785","ac0c5833","ac2e8ecf","ac3e2b04","ac605cbb","ad7e01d0","ae58858e","aee291af","af22c60d","af24b4cc","b0722778","b0f4d537","b15fca0b","b1fc8b8e","b20f7c8b","b457fec5","b4a43f3b","b7cb93ac","b7f8a4d8","ba9d41b8","baf41dbf","bb52a14b","bbb1b8b6","bc4146bd","be03b35f","bf32578f","bf699163","bf89d739","c074846d","c3202e5a","c35c1b4c","c48954c1","c62e2108","c658a4bd","c663677b","c6e1b8da","c7d4e6ad","c8b7cc0f","c92b942c","c97c0139","ca8de6ea","ca8f78db","cad67732","cb227835","ccd554ac","ce039d91","ce8d95cc","cf133acc","d19f7514","d282b262","d2acf2cb","d37a1ef5","d47aa2ff","d492a647","d4b1c2b1","d4c90558","d56f2372","d931c21c","d94c3b52","dc2e9a9d","dd2401ed","de493100","df8cc377","e133d23d","e1baa8a4","e1d2900e","e2092e0c","e21a174a","e345f17b","e4075551","e41c6fd3","e5790162","e5c44e8f","e619ca6e","e633a9e5","e66aafb8","e681b708","e69241bd","e74e1818","e760a62e","e7639916","e7a25a18","e7b06bea","e7dd8335","e872b94a","e88171ec","e99362f0","e9ac8c9e","e9bb6954","ea959feb","ea9794b1","ecaa0ec1","ed74f2f2","ed98d772","ef26cbf6","f0afb749","f0df5ff0","f21745ec","f3b10344","f3cdc58f","f3e62deb","f5aa3634","f5c89df1","f823c43c","f8be4b64","f9d67f8b","fafd9572","fb791726","fc754716","fd096ab6","fd4b2b02","fe9372f3","fea12743"],"datasetRevision":"3e9c9d1a8402aff82356815c106cecaea65cb7d9","datasetLastModified":"2026-06-04T16:43:53.000Z","datasetUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/tree/3e9c9d1a8402aff82356815c106cecaea65cb7d9"},{"id":"arc-agi-2-public-matched","label":"ARC-AGI-2 · matched public subset","taskClass":"Abstract grid reasoning","matchedTasks":108,"fullPublicTasks":120,"coveragePct":90,"taskIds":["0934a4d8","135a2760","136b0064","142ca369","16b78196","16de56c4","1818057f","1ae2feb7","20270e3b","20a9e565","221dfab4","247ef758","271d71e2","28a6681f","291dc1e1","2ba387bc","2c181942","2d0172a1","31f7f899","332f06d7","35ab12c3","36a08778","38007db0","3a25b0d8","3dc255db","3e6067c3","409aa875","446ef5d2","45a5af55","4a21e3da","4c3d4a41","4c416de3","4c7dc4dd","53fb4810","5545f144","581f7754","58490d8a","58f5dbd5","5961cc34","5dbc8537","62593bfd","64efde09","65b59efc","67e490f4","6e453dd6","71e489b6","7491f3cf","7666fa5d","78332cb0","7b0280bc","7b3084d4","7b5033c1","7b80bb43","7c66cb00","7ed72f31","800d221b","80a900e0","8698868d","88bcf3b4","89565ca0","898e7135","8b9c3697","8e5c0c38","8f215267","8f3a5a89","9385bd28","97d7923e","981571dc","9aaea919","a251c730","a25697e4","a32d8b75","a395ee82","a47bf94d","a6f40cea","aa4ec2a5","abc82100","b0039139","b10624e5","b5ca7ac4","b6f77b65","b99e7126","b9e38dc0","bf45cf4b","c4d067a0","c7f57c3e","cb2d8a2c","cbebaa4b","d35bdbdc","d59b0160","d8e07eb2","db0c5428","db695cfb","dbff022c","dd6b8c4b","de809cff","dfadab01","e12f9a14","e3721c99","e376de54","e8686506","e87109e9","edb79dae","eee78d87","f560132c","f931b4a8","faa9f03d","fc7cae8d"],"datasetRevision":"026789c1c12a4c34580a32e84dcaf5630d7e8f31","datasetLastModified":"2026-06-04T16:45:03.000Z","datasetUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/tree/026789c1c12a4c34580a32e84dcaf5630d7e8f31"}],"records":[{"modelId":"gpt-5-2-2025-12-11-thinking-high","benchmarkId":"arc-agi-1-public-matched","scorePct":90.2439024390244,"costPerTaskUsd":0.378617056402439,"matchedTasks":328,"sourceRunTasks":400,"sourceRawSummedCredit":361,"sourceRunCostUsd":160.811077,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-2-2025-12-11-thinking-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-2-2025-12-11-thinking-high/results.json","sourceFileSha256":"7a687310f13e092e4dd9d46011740ff2d7c6352e1723556cf31f1ae52b071ab3","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"00576224","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gpt-5-2-2025-12-11-thinking-high/00576224.json","requestSettings":{"background":true,"reasoning":{"effort":"high"},"max_output_tokens":196608},"sampleRunStart":"2025-12-08T08:18:36.237741+00:00","sampleRunEnd":"2025-12-08T08:19:59.702719+00:00","sampleApiModelId":"gpt-5.2-2025-12-11"}},{"modelId":"claude-opus-4-8-high","benchmarkId":"arc-agi-1-public-matched","scorePct":96.64634146341463,"costPerTaskUsd":0.5465530030487805,"matchedTasks":328,"sourceRunTasks":400,"sourceRawSummedCredit":382,"sourceRunCostUsd":248.9078950000002,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-8-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-8-high/results.json","sourceFileSha256":"b517adac111d87a91fd2a11dba49d4826851b473247a9f44167e54f594614241","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"00576224","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-8-high/00576224.json","requestSettings":{"max_tokens":128000,"stream":true,"thinking":{"type":"adaptive"},"extra_body":{"output_config":{"effort":"high"}}},"sampleRunStart":"2026-05-29T17:19:54.920909+00:00","sampleRunEnd":"2026-05-29T17:20:07.132243+00:00","sampleApiModelId":"claude-opus-4-8"}},{"modelId":"claude-opus-4-6-thinking-120K-high","benchmarkId":"arc-agi-1-public-matched","scorePct":98.17073170731707,"costPerTaskUsd":0.7797896189024391,"matchedTasks":328,"sourceRunTasks":400,"sourceRawSummedCredit":385,"sourceRunCostUsd":349.093095,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-6-thinking-120K-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-6-thinking-120K-high/results.json","sourceFileSha256":"aad50cd0f1f291cde14b535b39e4e4bb82e5b8ea1de03467b4d17c0225128110","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"00576224","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/claude-opus-4-6-thinking-120K-high/00576224.json","requestSettings":{"max_tokens":128000,"betas":["effort-2025-11-24","max-effort-2026-01-24"],"stream":true,"thinking":{"type":"enabled","budget_tokens":120000},"output_config":{"effort":"high"}},"sampleRunStart":"2026-02-04T05:34:09.497991+00:00","sampleRunEnd":"2026-02-04T05:34:32.389446+00:00","sampleApiModelId":"claude-opus-4-6"}},{"modelId":"gemini-3-flash-preview-thinking-high","benchmarkId":"arc-agi-1-public-matched","scorePct":88.8719512195122,"costPerTaskUsd":0.1348761493902439,"matchedTasks":328,"sourceRunTasks":330,"sourceRawSummedCredit":291.5,"sourceRunCostUsd":44.63808200000002,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-flash-preview-thinking-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-flash-preview-thinking-high/results.json","sourceFileSha256":"c15a849a79f2cd502521fdf5fcba228ea684ec013b78235b652ec7b7637bbaf8","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"00576224","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-flash-preview-thinking-high/00576224.json","requestSettings":{"max_output_tokens":64000,"thinking_config":{"thinking_level":"HIGH"},"automatic_function_calling":{"disable":true}},"sampleRunStart":"2025-12-15T19:39:19.924113+00:00","sampleRunEnd":"2025-12-15T19:40:01.985736+00:00","sampleApiModelId":"models/gemini-3-flash-preview"}},{"modelId":"gemini-3-1-pro-preview","benchmarkId":"arc-agi-1-public-matched","scorePct":96.95121951219512,"costPerTaskUsd":0.3610779512195122,"matchedTasks":328,"sourceRunTasks":398,"sourceRawSummedCredit":387,"sourceRunCostUsd":148.33994000000004,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/blob/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-1-pro-preview/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-1-pro-preview/results.json","sourceFileSha256":"338b1d4c43f3c0833ed18684407f75d3a50e61d6c77273cb6bb4f3b0b856efcc","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"00576224","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v1_public_eval/raw/3e9c9d1a8402aff82356815c106cecaea65cb7d9/gemini-3-1-pro-preview/00576224.json","requestSettings":{"max_output_tokens":64000,"automatic_function_calling":{"disable":true}},"sampleRunStart":"2026-02-09T04:59:38.426890+00:00","sampleRunEnd":"2026-02-09T05:00:06.310722+00:00","sampleApiModelId":"models/gemini-3.1-pro-preview"}},{"modelId":"gpt-5-2-2025-12-11-thinking-high","benchmarkId":"arc-agi-2-public-matched","scorePct":43.51851851851852,"costPerTaskUsd":1.3693686319444442,"matchedTasks":108,"sourceRunTasks":119,"sourceRawSummedCredit":47.5,"sourceRunCostUsd":177.35028499999996,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-2-2025-12-11-thinking-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-2-2025-12-11-thinking-high/results.json","sourceFileSha256":"ffceb300b993555906af6f6542b43ae8bb2fe6515a6f81bf17cec48d6dee7881","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"0934a4d8","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gpt-5-2-2025-12-11-thinking-high/0934a4d8.json","requestSettings":{"background":true,"reasoning":{"effort":"high"},"max_output_tokens":196608},"sampleRunStart":"2025-12-08T05:22:28.313425+00:00","sampleRunEnd":"2025-12-08T06:53:17.207161+00:00","sampleApiModelId":"gpt-5.2-2025-12-11"}},{"modelId":"claude-opus-4-8-high","benchmarkId":"arc-agi-2-public-matched","scorePct":80.40123456790123,"costPerTaskUsd":2.8625914814814815,"matchedTasks":108,"sourceRunTasks":120,"sourceRawSummedCredit":92.33333333333333,"sourceRunCostUsd":375.81268,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-8-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-8-high/results.json","sourceFileSha256":"194e9b5b18f611242f76be582e1029159bc57acce2d9dc58a330524acb47f318","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"0934a4d8","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-8-high/0934a4d8.json","requestSettings":{"max_tokens":128000,"stream":true,"thinking":{"type":"adaptive"},"extra_body":{"output_config":{"effort":"high"}}},"sampleRunStart":"2026-05-29T16:53:30.934096+00:00","sampleRunEnd":"2026-05-29T17:22:23.825979+00:00","sampleApiModelId":"claude-opus-4-8"}},{"modelId":"claude-opus-4-6-thinking-120K-high","benchmarkId":"arc-agi-2-public-matched","scorePct":83.179012345679,"costPerTaskUsd":3.5073681481481485,"matchedTasks":108,"sourceRunTasks":120,"sourceRawSummedCredit":94.83333333333333,"sourceRunCostUsd":457.10411500000004,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-6-thinking-120K-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-6-thinking-120K-high/results.json","sourceFileSha256":"72da4591aac9d15f28775f3d4ed513fa35bf35503ddc5be50c279798fe90ea47","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"0934a4d8","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/claude-opus-4-6-thinking-120K-high/0934a4d8.json","requestSettings":{"max_tokens":128000,"betas":["effort-2025-11-24","max-effort-2026-01-24"],"stream":true,"thinking":{"type":"enabled","budget_tokens":120000},"output_config":{"effort":"high"}},"sampleRunStart":"2026-02-04T09:35:54.976886+00:00","sampleRunEnd":"2026-02-04T10:07:27.678214+00:00","sampleApiModelId":"claude-opus-4-6"}},{"modelId":"gemini-3-flash-preview-thinking-high","benchmarkId":"arc-agi-2-public-matched","scorePct":37.808641975308646,"costPerTaskUsd":0.23239886111111113,"matchedTasks":108,"sourceRunTasks":120,"sourceRawSummedCredit":40.833333333333336,"sourceRunCostUsd":28.79222899999999,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-flash-preview-thinking-high/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-flash-preview-thinking-high/results.json","sourceFileSha256":"64657364dc566756c3fd35f17f4e9c5ac80c3beb6ce3e4a73f36c4c5f9b1535b","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"0934a4d8","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-flash-preview-thinking-high/0934a4d8.json","requestSettings":{"max_output_tokens":64000,"thinking_config":{"thinking_level":"HIGH"},"automatic_function_calling":{"disable":true}},"sampleRunStart":"2025-12-15T19:10:36.915424+00:00","sampleRunEnd":"2025-12-15T19:17:44.420032+00:00","sampleApiModelId":"models/gemini-3-flash-preview"}},{"modelId":"gemini-3-1-pro-preview","benchmarkId":"arc-agi-2-public-matched","scorePct":88.42592592592592,"costPerTaskUsd":0.9752575555555555,"matchedTasks":108,"sourceRunTasks":109,"sourceRawSummedCredit":96,"sourceRunCostUsd":106.70194399999998,"sourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/blob/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-1-pro-preview/results.json","rawDownloadUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-1-pro-preview/results.json","sourceFileSha256":"e6a56e4d9f7fb16f7731eece3db88c4e827e198509c5196ac39cd3613f44b030","metricOrigin":"computed from official publisher per-task records","latencySeconds":null,"configurationEvidence":{"sampleTaskId":"0934a4d8","sampleSourceUrl":"https://huggingface.co/datasets/arcprize/arc_agi_v2_public_eval/raw/026789c1c12a4c34580a32e84dcaf5630d7e8f31/gemini-3-1-pro-preview/0934a4d8.json","requestSettings":{"max_output_tokens":128000,"automatic_function_calling":{"disable":true}},"sampleRunStart":"2026-02-09T23:39:59.491362+00:00","sampleRunEnd":"2026-02-09T23:51:04.962717+00:00","sampleApiModelId":"models/gemini-3.1-pro-preview"}}],"benchmarkSampleCounts":{"arc-agi-1-public-matched":328,"arc-agi-2-public-matched":108}}