{"schema_version":1,"package_version":"1.3.0-rc.4","task_version":"1.3.0","source_commit":"d416d403eb49e879ecece7e7abfabff5bb0c75cd","source_url":"https://github.com/engrambio/bioevals/tree/main/projects/frontierSapien/lab_ready","archive":"sod1-ranking-v1.3.0-rc.4.zip","archive_sha256":"61fb73e8d875e5d763813521baf79d2be1c08c1e8215fd4108814cc7e1149b72","solver_archive":"sod1-ranking-v1.3.0-rc.4-solver-inputs.zip","solver_archive_sha256":"5e7d8f15bb33ab45adb6cfc8e0683e36ebe1265945151cc764ce40ddbf960e66","internal_files_verified":772,"historical_attempts_excluded":23,"formula":"max(0, spearman_rho)","tasks":[{"id":"sod1-abundance-n453-p01","endpoint":"Abundance","variants":453,"instruction":"protein-ranking/sod1-abundance-n453-p01/instruction.md","instruction_sha256":"0fa58e596ef169cbeaa96d4b95037b8c7c158c471b96b795c2271dd4841b2273","candidates_sha256":"1d02010d28ddd93556858b9d92c8e6a6d06d01bfdfb70850435e61e33bc72b3d"},{"id":"sod1-dimerization-n453-p01","endpoint":"Dimerization","variants":453,"instruction":"protein-ranking/sod1-dimerization-n453-p01/instruction.md","instruction_sha256":"5cefcdcd23ed1d7fe3ab15dfc922bff4ae7cec7017ef679e705ad4ed24a7a7ec","candidates_sha256":"1d02010d28ddd93556858b9d92c8e6a6d06d01bfdfb70850435e61e33bc72b3d"}],"summary":[{"task":"sod1-abundance-n453-p01","model":"us.openai.gpt-6-astra","n":3,"scores":[0.5894053934873639,0.6349480714253811,0.6028993099167868],"mean":0.6090842582765106,"sample_sd":0.023392820866685333,"valid_rankings":3,"invalid_rankings":0,"role":"fresh v1.3.0 generation","reasoning_effort":"max"},{"task":"sod1-abundance-n453-p01","model":"us.xai.grok-4.7","n":3,"scores":[0.0,0.3284558829794018,0.06433672466973261],"mean":0.13093086921637814,"sample_sd":0.1740600523723616,"valid_rankings":2,"invalid_rankings":1,"role":"fresh v1.3.0 generation","reasoning_effort":"high"},{"task":"sod1-abundance-n453-p01","model":"esm","n":1,"scores":[0.6383397520616136],"mean":0.6383397520616136,"sample_sd":null,"valid_rankings":1,"invalid_rankings":0,"role":"frozen supervised baseline","reasoning_effort":null},{"task":"sod1-dimerization-n453-p01","model":"us.openai.gpt-6-astra","n":3,"scores":[0.6327139305723981,0.6819312519218104,0.6491178971115336],"mean":0.654587693201914,"sample_sd":0.02506043060594897,"valid_rankings":3,"invalid_rankings":0,"role":"fresh v1.3.0 generation","reasoning_effort":"max"},{"task":"sod1-dimerization-n453-p01","model":"us.xai.grok-4.7","n":3,"scores":[0.14050069953251762,0.0,0.0],"mean":0.046833566510839204,"sample_sd":0.08111811669642978,"valid_rankings":2,"invalid_rankings":1,"role":"fresh v1.3.0 generation","reasoning_effort":"high"},{"task":"sod1-dimerization-n453-p01","model":"esm","n":1,"scores":[0.5896984251933944],"mean":0.5896984251933944,"sample_sd":null,"valid_rankings":1,"invalid_rankings":0,"role":"frozen supervised baseline","reasoning_effort":null}],"runs":[{"trial":"sod1-abundance-n453-p01__ARDR33z","task":"sod1-abundance-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.0,"rho":null,"status":"invalid_submission","missing_ids":["P229","P407"],"input_tokens":46895,"output_tokens":48826,"source":"evidence/current/sod1-abundance-n453-p01__ARDR33z/result.json","source_sha256":"60e85736c78b797d28ebcb74b62b82f8e9ff654f9530405676062e357da16169","request_sha256":"16f522a12b7f419c5456f06567fa16a5f3e76e4dee76157d99b49abe692c9ad7","replay_source":"results/regrades/sod1-abundance-n453-p01__ARDR33z/metrics.json","replay_sha256":"5a61f5fc272e8d47974053ae42ea8b47dcf247d2fba111ba85f5e4c00c127750"},{"trial":"sod1-abundance-n453-p01__FGCw4cr","task":"sod1-abundance-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.3284558829794018,"rho":0.3284558829794018,"status":"graded","missing_ids":[],"input_tokens":46895,"output_tokens":46373,"source":"evidence/current/sod1-abundance-n453-p01__FGCw4cr/result.json","source_sha256":"2376a50d6a96964459fd1ad9a9be17f59dddd852f35a8cc6126aec02788bdf94","request_sha256":"16f522a12b7f419c5456f06567fa16a5f3e76e4dee76157d99b49abe692c9ad7","replay_source":"results/regrades/sod1-abundance-n453-p01__FGCw4cr/metrics.json","replay_sha256":"60e906bbaec8fe2c2ff5a8da56dd78af2addc3a273951cc1917f3eea59bb8537"},{"trial":"sod1-abundance-n453-p01__RdoE5pm","task":"sod1-abundance-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.5894053934873639,"rho":0.5894053934873639,"status":"graded","missing_ids":[],"input_tokens":45675,"output_tokens":29984,"source":"evidence/current/sod1-abundance-n453-p01__RdoE5pm/result.json","source_sha256":"4aa05b85028e408c385a7c53673bcc0877a36df948da8ba337f93d205ee113b7","request_sha256":"5d2314cac59f052078fee7dd604bfd3c498e28246d0c3f220f851ada530eb8f5","replay_source":"results/regrades/sod1-abundance-n453-p01__RdoE5pm/metrics.json","replay_sha256":"52b596efea4d900a64b680d3b53ab6cc39fc80578d8999b21362307f323a15fe"},{"trial":"sod1-abundance-n453-p01__W2zsUVT","task":"sod1-abundance-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.06433672466973261,"rho":0.06433672466973261,"status":"graded","missing_ids":[],"input_tokens":46895,"output_tokens":31164,"source":"evidence/current/sod1-abundance-n453-p01__W2zsUVT/result.json","source_sha256":"33b0317ba9c35dd8ee6277a58393badcceabe7d25d8fdc7da9e24a8b74a87411","request_sha256":"16f522a12b7f419c5456f06567fa16a5f3e76e4dee76157d99b49abe692c9ad7","replay_source":"results/regrades/sod1-abundance-n453-p01__W2zsUVT/metrics.json","replay_sha256":"0080ed7e363742e82b758ad8df22a2e5656e82c4c2ccaf1065eb17ee2a28c0c9"},{"trial":"sod1-abundance-n453-p01__oeVhHva","task":"sod1-abundance-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.6349480714253811,"rho":0.6349480714253811,"status":"graded","missing_ids":[],"input_tokens":45675,"output_tokens":36190,"source":"evidence/current/sod1-abundance-n453-p01__oeVhHva/result.json","source_sha256":"bc16db4ac7b1064371ffe624770a49ffbb0842c88ade607cd75a67f11dcfa2e0","request_sha256":"5d2314cac59f052078fee7dd604bfd3c498e28246d0c3f220f851ada530eb8f5","replay_source":"results/regrades/sod1-abundance-n453-p01__oeVhHva/metrics.json","replay_sha256":"b6c1f306db75772f5d971ae749f4b796f3a47269eaab1a2b671e2d006add8e55"},{"trial":"sod1-abundance-n453-p01__wyy9DLu","task":"sod1-abundance-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.6028993099167868,"rho":0.6028993099167868,"status":"graded","missing_ids":[],"input_tokens":45675,"output_tokens":33268,"source":"evidence/current/sod1-abundance-n453-p01__wyy9DLu/result.json","source_sha256":"10252b2771d04dd45150e62b124c4f4f6ee25d4c5a9fb2241e33762185c4df75","request_sha256":"5d2314cac59f052078fee7dd604bfd3c498e28246d0c3f220f851ada530eb8f5","replay_source":"results/regrades/sod1-abundance-n453-p01__wyy9DLu/metrics.json","replay_sha256":"cbe00e89954af5107763f37dd4597931f21b5deb17197729c8cd88c4782b5730"},{"trial":"sod1-dimerization-n453-p01__HuZbF2e","task":"sod1-dimerization-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.6327139305723981,"rho":0.6327139305723981,"status":"graded","missing_ids":[],"input_tokens":45700,"output_tokens":34899,"source":"evidence/current/sod1-dimerization-n453-p01__HuZbF2e/result.json","source_sha256":"7e336580ef02ac56626516a235148c9042ed34da86fb2cf395d18ce892721030","request_sha256":"bd828c7a553d414ecab5683c9b58d624ec24e492cea950ac1db349f4bb676fb3","replay_source":"results/regrades/sod1-dimerization-n453-p01__HuZbF2e/metrics.json","replay_sha256":"107c2e5221f1ec6c96fb949f2a3bf8c0c8287ebcb5e03d0cc392e8c107c86f95"},{"trial":"sod1-dimerization-n453-p01__KdGz5zc","task":"sod1-dimerization-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.14050069953251762,"rho":0.14050069953251762,"status":"graded","missing_ids":[],"input_tokens":46919,"output_tokens":54229,"source":"evidence/current/sod1-dimerization-n453-p01__KdGz5zc/result.json","source_sha256":"e7c78dbe8e855a81fc108042b7d13ca178f311b417c820df9cb4c9938ea08b0b","request_sha256":"dc5b8b87f3fb6226bc4cd49674f90227a293c5b7a2fce8f089efe656337f5fb2","replay_source":"results/regrades/sod1-dimerization-n453-p01__KdGz5zc/metrics.json","replay_sha256":"aa3df7b5b337d2ec006a5b414a5a3f4551e21c16ede6890ae31ee625b7cd099b"},{"trial":"sod1-dimerization-n453-p01__brYxqMd","task":"sod1-dimerization-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.0,"rho":-0.0016658917032267824,"status":"graded","missing_ids":[],"input_tokens":46919,"output_tokens":26495,"source":"evidence/current/sod1-dimerization-n453-p01__brYxqMd/result.json","source_sha256":"0e255c8cae3a126aedb7d0ceaf899a605cb8cd03e314ce97df8d8c7b2593cc58","request_sha256":"dc5b8b87f3fb6226bc4cd49674f90227a293c5b7a2fce8f089efe656337f5fb2","replay_source":"results/regrades/sod1-dimerization-n453-p01__brYxqMd/metrics.json","replay_sha256":"4127c92b6f1db13d89040810fac50e5cab3bef68b17876a703651b17962af14d"},{"trial":"sod1-dimerization-n453-p01__iditfZ5","task":"sod1-dimerization-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.6819312519218104,"rho":0.6819312519218104,"status":"graded","missing_ids":[],"input_tokens":45700,"output_tokens":37627,"source":"evidence/current/sod1-dimerization-n453-p01__iditfZ5/result.json","source_sha256":"cfc3c6669be3bcea057712d12c6be0d756699acf1d073ea83043b201b09b6998","request_sha256":"bd828c7a553d414ecab5683c9b58d624ec24e492cea950ac1db349f4bb676fb3","replay_source":"results/regrades/sod1-dimerization-n453-p01__iditfZ5/metrics.json","replay_sha256":"a982fd010f7a004dcd1302b3b04bb092d17ecd2206eee00b4fcbc250d70e4976"},{"trial":"sod1-dimerization-n453-p01__xyV5gfa","task":"sod1-dimerization-n453-p01","model":"us.xai.grok-4.7","effort":"high","max_output_tokens":128000,"score":0.0,"rho":null,"status":"invalid_submission","missing_ids":["P339"],"input_tokens":46919,"output_tokens":55415,"source":"evidence/current/sod1-dimerization-n453-p01__xyV5gfa/result.json","source_sha256":"46f4c813d37ce6cc65a08e1a4639d2eaca0500a92a5a3d784d4d639e7f5b12c7","request_sha256":"dc5b8b87f3fb6226bc4cd49674f90227a293c5b7a2fce8f089efe656337f5fb2","replay_source":"results/regrades/sod1-dimerization-n453-p01__xyV5gfa/metrics.json","replay_sha256":"89d15c32ff2d88864ce30e30a1fa19bb0ef1814c18c869a7181fda13f9c057bf"},{"trial":"sod1-dimerization-n453-p01__ymwMJe6","task":"sod1-dimerization-n453-p01","model":"us.openai.gpt-6-astra","effort":"max","max_output_tokens":128000,"score":0.6491178971115336,"rho":0.6491178971115336,"status":"graded","missing_ids":[],"input_tokens":45700,"output_tokens":37090,"source":"evidence/current/sod1-dimerization-n453-p01__ymwMJe6/result.json","source_sha256":"3c6b5f9e740e1b7decfb1fad6e813f643616d7c756a9aea9a71e2ddd6b17ebf5","request_sha256":"bd828c7a553d414ecab5683c9b58d624ec24e492cea950ac1db349f4bb676fb3","replay_source":"results/regrades/sod1-dimerization-n453-p01__ymwMJe6/metrics.json","replay_sha256":"2b365a636dca6bb7901e2f147c269d10c82d88adc179bda82f682f0a89b5f6a2"}],"baseline":[{"task":"sod1-abundance-n453-p01","score":0.6383397520616136,"rho":0.6383397520616136,"role":"Frozen supervised ESM-2 + ridge baseline, regraded","trial":"historical-sod1-abundance-n453-p01-esm","source":"results/regrades/historical-sod1-abundance-n453-p01-esm/metrics.json","source_sha256":"8eab55216e45de93b397973e182788f99dac637979ab3359540e76cb5aba9be8"},{"task":"sod1-dimerization-n453-p01","score":0.5896984251933944,"rho":0.5896984251933944,"role":"Frozen supervised ESM-2 + ridge baseline, regraded","trial":"historical-sod1-dimerization-n453-p01-esm","source":"results/regrades/historical-sod1-dimerization-n453-p01-esm/metrics.json","source_sha256":"e761cc2348c98b569cede6ede4edc2b8d4789422426017d0d766eab35dc13e21"}],"current":{"current_initiated_requests":12,"current_returned_rankings":12,"current_unreturned_requests":0,"current_valid_rankings":10,"current_invalid_rankings":2},"validation":{"checked_at_utc":"2026-10-28T01:27:29.764312+00:00","project_gate":"PASS","adapter_tests_passed":11,"saved_answer_replays_passed":28,"controls_passed":44,"isolation_receipts_passed":12,"source_labels_checked":906,"current_task_prompt_exact_match":true},"sources":{"QC_REPORT.md":"a354b93572b23435d142e85da2a631a99ed4594c3fed2cd108ba0dafdf9b6f25","ACCEPTANCE.md":"1a0756c070dda5dfd4594e2a6f6791c04595c315e0f08e64836074034bc3b372","VALIDATION.json":"5b7e9e822dfcbb22b75305607339ebc439bda341183f0155bda3e995e4a5e0c5","results/summary.json":"a9bfd1bb77347dc898ae74f738fc999f7f1d3c1df388d60ef2e1a9387f06336b"}}
