{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,20]],"date-time":"2026-08-20T19:31:38Z","timestamp":1787254298362,"version":"build-2736575974"},"reference-count":17,"publisher":"IEEE","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Office of Science"},{"name":"High Energy Physics"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1109\/ipdpsw71298.2026.00112","type":"proceedings-article","created":{"date-parts":[[2026,8,19]],"date-time":"2026-08-19T19:11:00Z","timestamp":1787166660000},"page":"677-682","source":"Crossref","is-referenced-by-count":0,"title":["CelloAI Benchmarks: Toward Repeatable Evaluation of AI Assistants"],"prefix":"10.1109","author":[{"given":"Mohammad","family":"Atif","sequence":"first","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kriti","family":"Chopra","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fang-Ying","family":"Tsai","sequence":"additional","affiliation":[{"name":"Stony Brook University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ozgur O.","family":"Kilic","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tianle","family":"Wang","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhihua","family":"Dong","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Douglas","family":"Benjamin","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Charles","family":"Leggett","sequence":"additional","affiliation":[{"name":"Lawrence Berkeley National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Meifeng","family":"Lin","sequence":"additional","affiliation":[{"name":"Brookhaven National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paolo","family":"Calafiura","sequence":"additional","affiliation":[{"name":"Lawrence Berkeley National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Salman","family":"Habib","sequence":"additional","affiliation":[{"name":"Argonne National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Leveraging LLMs for legacy code modernization: Challenges and opportunities for LLM-generated documentation","author":"Diggs","year":"2024","journal-title":"arXiv preprint arXiv:2411.14971"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639187"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1002\/smr.2723"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3815572.3815751"},{"key":"ref5","article-title":"Swe-bench: Can language models resolve real-world github issues?","author":"Jimenez","year":"2023","journal-title":"arXiv preprint arXiv:2310.06770"},{"key":"ref6","article-title":"Livecodebench: Holistic and contamination free evaluation of large language models for code","author":"Jain","year":"2024","journal-title":"arXiv preprint arXiv:2403.07974"},{"key":"ref7","article-title":"Bigcodebench: Benchmarking code generation with diverse function calls and complex instructions","author":"Zhuo","year":"2024","journal-title":"arXiv preprint arXiv:2406.15877"},{"key":"ref8","article-title":"Program synthesis with large language models","author":"Austin","year":"2021","journal-title":"arXiv preprint arXiv:2108.07732"},{"key":"ref9","article-title":"The new atlas fast calorimeter simulation","author":"Hasib","year":"2017","journal-title":"ATL-COM-SOFT-2017-036, Tech. Rep."},{"key":"ref10","article-title":"Athena"},{"key":"ref11","article-title":"How efficient is llm-generated code\u0192 a rigorous & high-standard benchmark","author":"Qiu","year":"2024","journal-title":"arXiv preprint arXiv:2406.06647"},{"key":"ref12","article-title":"Kernelbench: Can llms write efficient gpu kernels\u0192","author":"Ouyang","year":"2025","journal-title":"arXiv preprint arXiv:2502.10517"},{"key":"ref13","article-title":"Evaluating application characteristics for GPU portability layer selection","author":"Atif","year":"2026","journal-title":"arXiv preprint arXiv:2601.17526"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1088\/1742-6596\/3206\/1\/012036"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1051\/epjconf\/202429511018"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/WACV45572.2020.9093523"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-acl.177"}],"event":{"name":"2026 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)","location":"New Orleans, LA, USA","start":{"date-parts":[[2026,5,25]]},"end":{"date-parts":[[2026,5,29]]}},"container-title":["2026 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11651886\/11651887\/11651933.pdf?arnumber=11651933","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,20]],"date-time":"2026-08-20T19:07:39Z","timestamp":1787252859000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11651933\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":17,"URL":"https:\/\/doi.org\/10.1109\/ipdpsw71298.2026.00112","relation":{},"subject":[],"published":{"date-parts":[[2026,5]]}}}