{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,3]],"date-time":"2026-05-03T05:55:57Z","timestamp":1777787757120,"version":"3.51.4"},"reference-count":41,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["CMMI2153855"],"award-info":[{"award-number":["CMMI2153855"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/access.2026.3685519","type":"journal-article","created":{"date-parts":[[2026,4,20]],"date-time":"2026-04-20T20:05:45Z","timestamp":1776715545000},"page":"61784-61808","source":"Crossref","is-referenced-by-count":0,"title":["SimBench: A Framework for Evaluating and Diagnosing LLM-Based Digital-Twin Generation for Multi-Physics Simulation"],"prefix":"10.1109","volume":"14","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8246-8018","authenticated-orcid":false,"given":"Jingquan","family":"Wang","sequence":"first","affiliation":[{"name":"University of Wisconsin–Madison, Madison, WI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andrew","family":"Negrut","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Rice University, Houston, TX, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-5478-5372","authenticated-orcid":false,"given":"Hongyu","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Wisconsin–Madison, Madison, WI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-1903-274X","authenticated-orcid":false,"given":"Harry","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Wisconsin–Madison, Madison, WI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1565-2784","authenticated-orcid":false,"given":"Dan","family":"Negrut","sequence":"additional","affiliation":[{"name":"University of Wisconsin–Madison, Madison, WI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Codebleu: A method for automatic evaluation of code synthesis","author":"Ren","year":"2020","journal-title":"arXiv:2009.10297"},{"key":"ref2","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Text Summarization Branches Out","author":"Lin","year":"2004"},{"key":"ref3","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021","journal-title":"arXiv:2107.03374"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-81166-2_50"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-40361-8_2"},{"key":"ref6","volume-title":"Chrono: An Open Source Framework for the Physics-Based Simulation of Dynamic Systems","year":"2020"},{"key":"ref7","volume-title":"Chrono: An Open Source Framework for the Physics-Based Simulation of Dynamic Systems","year":"2020"},{"key":"ref8","volume-title":"Pychrono 7.0.0 Download Statistics","year":"2022"},{"key":"ref9","volume-title":"Chrono User Forum","year":"2023"},{"key":"ref10","first-page":"1","article-title":"ROS: An open-source robot operating system","volume-title":"Proc. ICRA Workshop Open Source Softw.","author":"Quigley"},{"key":"ref11","volume-title":"Functional Mock-Up Interface Specification (Version: 3.0)","year":"2022"},{"key":"ref12","volume-title":"Foundational Research Gaps and Future Directions for Digital Twins","author":"Willcox","year":"2023"},{"key":"ref13","volume-title":"OpenFOAM: An Open Source Free CFD Code","year":"2024"},{"key":"ref14","volume-title":"PyBullet, a Python Module for Physics Simulation for Games, Robotics and Machine Learning","author":"Coumans","year":"2016"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ETFA61755.2024.10710900"},{"key":"ref16","article-title":"Large language models for explainable decisions in dynamic digital twins","author":"Zhang","year":"2024","journal-title":"arXiv:2405.14411"},{"key":"ref17","first-page":"1","article-title":"Rule based rewards for fine-grained llm safety","volume-title":"Proc. ICML Next Gener. AI Saf. Workshop","author":"Mu"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"ref19","article-title":"CodeT: Code generation with generated tests","author":"Chen","year":"2022","journal-title":"arXiv:2207.10397"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref21","article-title":"Program synthesis with large language models","author":"Austin","year":"2021","journal-title":"arXiv:2108.07732"},{"key":"ref22","first-page":"28","article-title":"Execution-based evaluation for data science code generation models","volume-title":"Proc. 4th Workshop Data Sci. Hum.-Loop","author":"Huang"},{"key":"ref23","article-title":"DS-1000: A natural and reliable benchmark for data science code generation","author":"Lai","year":"2022","journal-title":"arXiv:2211.11501"},{"key":"ref24","first-page":"1","article-title":"Measuring coding challenge competence with APPS","volume-title":"Proc. NeurIPS","author":"Hendrycks"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.89"},{"key":"ref26","article-title":"MINT: Evaluating LLMs in multi-turn interaction with tools and language feedback","author":"Wang","year":"2023","journal-title":"arXiv:2309.10691"},{"key":"ref27","article-title":"PyBench: Evaluating LLM agent on various real-world coding tasks","author":"Zhang","year":"2024","journal-title":"arXiv:2407.16732"},{"key":"ref28","article-title":"Judging LLM-as-a-judge with MT-bench and chatbot arena","author":"Zheng","year":"2023","journal-title":"arXiv:2306.05685"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i4.20363"},{"key":"ref30","article-title":"Large language models are not fair evaluators","author":"Wang","year":"2023","journal-title":"arXiv:2305.17926"},{"key":"ref31","article-title":"Self-rewarding language models","author":"Yuan","year":"2024","journal-title":"arXiv:2401.10020"},{"key":"ref32","article-title":"A survey of deep learning models for structural code understanding","author":"Wu","year":"2022","journal-title":"arXiv:2205.01293"},{"issue":"3","key":"ref33","doi-asserted-by":"crossref","first-page":"368","DOI":"10.3390\/sym12030368","article-title":"Implementing a novel use of multicriteria decision analysis to select IIoT platforms for smart manufacturing","volume":"12","author":"Contreras-Masse","year":"2020","journal-title":"Symmetry"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-22167-5_8"},{"key":"ref35","first-page":"21558","article-title":"Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Liu"},{"key":"ref36","article-title":"BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions","author":"Yue Zhuo","year":"2024","journal-title":"arXiv:2406.15877"},{"key":"ref37","volume-title":"User Manual","year":"2017"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"ref39","volume-title":"PyBullet: A Python Module for Physics Simulation","author":"Coumans","year":"2023"},{"key":"ref40","volume-title":"ISAAC SDK: NVIDIA\u2019s Robotics Platform","year":"2023"},{"key":"ref41","volume-title":"A 3D Multi-Robot Simulator With Dynamics"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/6287639\/11323511\/11488197-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/11323511\/11488197.pdf?arnumber=11488197","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T19:51:55Z","timestamp":1777492315000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11488197\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":41,"URL":"https:\/\/doi.org\/10.1109\/access.2026.3685519","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}