{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T15:49:49Z","timestamp":1781884189249,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","funder":[{"name":"National Science Foundation (NSF)","award":["2419880"],"award-info":[{"award-number":["2419880"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,30]]},"DOI":"10.1145\/3716368.3735171","type":"proceedings-article","created":{"date-parts":[[2025,6,27]],"date-time":"2025-06-27T13:58:23Z","timestamp":1751032703000},"page":"427-434","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["HWFixBench: Benchmarking Tools for Hardware Understanding and Fault Repair"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9623-6522","authenticated-orcid":false,"given":"Weimin","family":"Fu","sequence":"first","affiliation":[{"name":"Mike Wiegers Department of Electrical and Computer Engineering, Kansas State University, Manhattan, Kansas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-6343-6941","authenticated-orcid":false,"given":"Shijie","family":"Li","sequence":"additional","affiliation":[{"name":"School of Cyber Science and Technology, University of Science and Technology of China, Hefei, Anhui, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8791-0597","authenticated-orcid":false,"given":"Yier","family":"Jin","sequence":"additional","affiliation":[{"name":"School of Cyber Science and Technology, University of Science and Technology of China, Hefei, Anhui, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9896-9407","authenticated-orcid":false,"given":"Xiaolong","family":"Guo","sequence":"additional","affiliation":[{"name":"Mike Wiegers Department of Electrical and Computer Engineering, Kansas State University, Manhattan, Kansas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,6,29]]},"reference":[{"key":"e_1_3_3_1_2_2","doi-asserted-by":"publisher","unstructured":"Manar Abdelatty Jingxiao Ma et\u00a0al. 2024. MetRex: A Benchmark for Verilog Code Metric Reasoning Using LLMs. CoRR abs\/2411.03471 (2024) 7. 10.48550\/ARXIV.2411.03471 arXiv:https:\/\/arXiv.org\/abs\/2411.03471","DOI":"10.48550\/ARXIV.2411.03471"},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","unstructured":"Marah\u00a0I Abdin Jyoti Aneja et\u00a0al. 2024. Phi-4 Technical Report. CoRR abs\/2412.08905 (2024) 36. 10.48550\/ARXIV.2412.08905 arXiv:https:\/\/arXiv.org\/abs\/2412.08905","DOI":"10.48550\/ARXIV.2412.08905"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","unstructured":"Marah\u00a0I Abdin Sam\u00a0Ade Jacobs et\u00a0al. 2024. Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone. CoRR abs\/2404.14219 (2024) 24. 10.48550\/ARXIV.2404.14219 arXiv:https:\/\/arXiv.org\/abs\/2404.14219","DOI":"10.48550\/ARXIV.2404.14219"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","unstructured":"Jason Blocklove Siddharth Garg et\u00a0al. 2024. Evaluating LLMs for Hardware Design and Test. CoRR abs\/2405.02326 (2024) 6. 10.48550\/ARXIV.2405.02326 arXiv:https:\/\/arXiv.org\/abs\/2405.02326","DOI":"10.48550\/ARXIV.2405.02326"},{"key":"e_1_3_3_1_6_2","first-page":"15","volume-title":"The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024","author":"Chan Chi-Min","year":"2024","unstructured":"Chi-Min Chan, Weize Chen, et\u00a0al. 2024. ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate. In The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024. OpenReview.net, Vienna, Austria, 15. https:\/\/openreview.net\/forum?id=FQepisCUWu"},{"key":"e_1_3_3_1_7_2","doi-asserted-by":"publisher","unstructured":"Fan Cui Chenyang Yin et\u00a0al. 2024. OriGen: Enhancing RTL Code Generation with Code-to-Code Augmentation and Self-Reflection. CoRR abs\/2407.16237 (2024) 9. 10.48550\/arXiv.2407.16237","DOI":"10.48550\/arXiv.2407.16237"},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","unstructured":"DeepSeek-AI Daya Guo et\u00a0al. 2025. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. CoRR abs\/2501.12948 (2025) 22. 10.48550\/ARXIV.2501.12948 arXiv:https:\/\/arXiv.org\/abs\/2501.12948","DOI":"10.48550\/ARXIV.2501.12948"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"publisher","unstructured":"Matthew DeLorenzo Vasudev Gohil et\u00a0al. 2024. CreativEval: Evaluating Creativity of LLM-Based Hardware Code Generation. CoRR abs\/2404.08806 (2024) 5. 10.48550\/ARXIV.2404.08806 arXiv:https:\/\/arXiv.org\/abs\/2404.08806","DOI":"10.48550\/ARXIV.2404.08806"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","unstructured":"Abhimanyu Dubey Abhinav Jauhri et\u00a0al. 2024. The Llama 3 Herd of Models. CoRR abs\/2407.21783 (2024) 92. 10.48550\/ARXIV.2407.21783 arXiv:https:\/\/arXiv.org\/abs\/2407.21783","DOI":"10.48550\/ARXIV.2407.21783"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","unstructured":"Weimin Fu Shijie Li et\u00a0al. 2025. A Generalize Hardware Debugging Approach for Large Language Models Semi-Synthetic Datasets. IEEE Transactions on Circuits and Systems I: Regular Papers 72 2 (2025) 623\u2013636. 10.1109\/TCSI.2024.3487486","DOI":"10.1109\/TCSI.2024.3487486"},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1109\/ASIANHOST59942.2023.10409307"},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCD63220.2024.00033"},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"publisher","unstructured":"Daya Guo Qihao Zhu et\u00a0al. 2024. DeepSeek-Coder: When the Large Language Model Meets Programming - The Rise of Code Intelligence. CoRR abs\/2401.14196 (2024) 23. 10.48550\/ARXIV.2401.14196 arXiv:https:\/\/arXiv.org\/abs\/2401.14196","DOI":"10.48550\/ARXIV.2401.14196"},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"publisher","unstructured":"Chia-Tung Ho Haoxing Ren et\u00a0al. 2024. VerilogCoder: Autonomous Verilog Coding Agents with Graph-based Planning and Abstract Syntax Tree (AST)-based Waveform Tracing Tool. CoRR abs\/2408.08927 (2024) 30. 10.48550\/ARXIV.2408.08927 arXiv:https:\/\/arXiv.org\/abs\/2408.08927","DOI":"10.48550\/ARXIV.2408.08927"},{"key":"e_1_3_3_1_16_2","first-page":"51","volume-title":"The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024","author":"Jimenez Carlos\u00a0E.","year":"2024","unstructured":"Carlos\u00a0E. Jimenez, John Yang, et\u00a0al. 2024. SWE-bench: Can Language Models Resolve Real-world Github Issues?. In The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024. OpenReview.net, Vienna, Austria, 51. https:\/\/openreview.net\/forum?id=VTF8yNQM66"},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"publisher","unstructured":"Minwoo Kang Mingjie Liu et\u00a0al. 2024. FVEval: Understanding Language Model Capabilities in Formal Verification of Digital Hardware. CoRR abs\/2410.23299 (2024) 42. 10.48550\/ARXIV.2410.23299 arXiv:https:\/\/arXiv.org\/abs\/2410.23299","DOI":"10.48550\/ARXIV.2410.23299"},{"key":"e_1_3_3_1_18_2","unstructured":"Raymond Li Loubna\u00a0Ben Allal et\u00a0al. 2023. StarCoder: may the source be with you! Trans. Mach. Learn. Res. 2023 (2023) 43. https:\/\/openreview.net\/forum?id=KoFOg41haE"},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCAD57390.2023.10323812"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"publisher","unstructured":"Mingjie Liu Yun-Da Tsai et\u00a0al. 2024. CraftRTL: High-quality Synthetic Data Generation for Verilog Code Models with Correct-by-Construction Non-Textual Representations and Targeted Code Repair. CoRR abs\/2409.12993 (2024) 46. 10.48550\/ARXIV.2409.12993 arXiv:https:\/\/arXiv.org\/abs\/2409.12993","DOI":"10.48550\/ARXIV.2409.12993"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","unstructured":"Shang Liu Wenji Fang et\u00a0al. 2024. RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique. IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems Early Access (2024) 1\u20131. 10.1109\/TCAD.2024.3483089","DOI":"10.1109\/TCAD.2024.3483089"},{"key":"e_1_3_3_1_22_2","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2023.EMNLP-MAIN.153"},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.1109\/ASP-DAC58780.2024.10473904"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","unstructured":"Ruiyang Ma Yuxin Yang et\u00a0al. 2024. VerilogReader: LLM-Aided Hardware Test Generation. CoRR abs\/2406.04373 (2024) 5. 10.48550\/ARXIV.2406.04373 arXiv:https:\/\/arXiv.org\/abs\/2406.04373","DOI":"10.48550\/ARXIV.2406.04373"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","unstructured":"Thomas Mesnard Cassidy Hardin et\u00a0al. 2024. Gemma: Open Models Based on Gemini Research and Technology. CoRR abs\/2403.08295 (2024) 17. 10.48550\/ARXIV.2403.08295 arXiv:https:\/\/arXiv.org\/abs\/2403.08295","DOI":"10.48550\/ARXIV.2403.08295"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"publisher","unstructured":"Zhendong Mi Renming Zheng et\u00a0al. 2024. PromptV: Leveraging LLM-powered Multi-Agent Prompting for High-quality Verilog Generation. CoRR abs\/2412.11014 (2024) 5. 10.48550\/ARXIV.2412.11014 arXiv:https:\/\/arXiv.org\/abs\/2412.11014","DOI":"10.48550\/ARXIV.2412.11014"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","unstructured":"Bardia Nadimi and Hao Zheng. 2024. A Multi-Expert Large Language Model Architecture for Verilog Code Generation. CoRR abs\/2404.08029 (2024) 5. 10.48550\/ARXIV.2404.08029 arXiv:https:\/\/arXiv.org\/abs\/2404.08029","DOI":"10.48550\/ARXIV.2404.08029"},{"key":"e_1_3_3_1_28_2","first-page":"9","volume-title":"Forty-first International Conference on Machine Learning, ICML 2024, Vienna, Austria, July 21-27, 2024","author":"Pei Zehua","year":"2024","unstructured":"Zehua Pei, Hui-Ling Zhen, et\u00a0al. 2024. BetterV: Controlled Verilog Generation with Discriminative Guidance. In Forty-first International Conference on Machine Learning, ICML 2024, Vienna, Austria, July 21-27, 2024. OpenReview.net, Vienna, Austria, 9. https:\/\/openreview.net\/forum?id=jKnW7r7de1"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","unstructured":"Ruidi Qiu Grace\u00a0Li Zhang et\u00a0al. 2024. CorrectBench: Automatic Testbench Generation with Functional Self-Correction using LLMs for HDL Design. CoRR abs\/2411.08510 (2024) 7. 10.48550\/ARXIV.2411.08510 arXiv:https:\/\/arXiv.org\/abs\/2411.08510","DOI":"10.48550\/ARXIV.2411.08510"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","unstructured":"Morgane Rivi\u00e8re Shreya Pathak et\u00a0al. 2024. Gemma 2: Improving Open Language Models at a Practical Size. CoRR abs\/2408.00118 (2024) 21. 10.48550\/ARXIV.2408.00118 arXiv:https:\/\/arXiv.org\/abs\/2408.00118","DOI":"10.48550\/ARXIV.2408.00118"},{"key":"e_1_3_3_1_31_2","unstructured":"Pat Rondon Renyao Wei et\u00a0al. 2025. Evaluating Agent-based Program Repair at Google. arxiv:cs.SE\/2501.07531https:\/\/arxiv.org\/abs\/2501.07531"},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.645"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","unstructured":"Diomidis Spinellis. 2012. Git. IEEE Softw. 29 3 (2012) 100\u2013101. 10.1109\/MS.2012.61","DOI":"10.1109\/MS.2012.61"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.23919\/DATE56975.2023.10137086"},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","unstructured":"Shailja Thakur Baleegh Ahmad et\u00a0al. 2024. VeriGen: A Large Language Model for Verilog Code Generation. ACM Trans. Design Autom. Electr. Syst. 29 3 (2024) 46:1\u201346:31. 10.1145\/3643681","DOI":"10.1145\/3643681"},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","unstructured":"Kiran Thorat Jiahui Zhao et\u00a0al. 2023. Advanced Large Language Model (LLM)-Driven Verilog Development: Enhancing Power Performance and Area Optimization in Code Synthesis. CoRR abs\/2312.01022 (2023) 8. 10.48550\/ARXIV.2312.01022 arXiv:https:\/\/arXiv.org\/abs\/2312.01022","DOI":"10.48550\/ARXIV.2312.01022"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.1118"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","unstructured":"Hugo Touvron Louis Martin et\u00a0al. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. CoRR abs\/2307.09288 (2023) 77. 10.48550\/ARXIV.2307.09288 arXiv:https:\/\/arXiv.org\/abs\/2307.09288","DOI":"10.48550\/ARXIV.2307.09288"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","unstructured":"Yun-Da Tsai Mingjie Liu et\u00a0al. 2023. RTLFixer: Automatically Fixing RTL Syntax Errors with Large Language Models. CoRR abs\/2311.16543 (2023) 7. 10.48550\/ARXIV.2311.16543 arXiv:https:\/\/arXiv.org\/abs\/2311.16543","DOI":"10.48550\/ARXIV.2311.16543"},{"key":"e_1_3_3_1_40_2","doi-asserted-by":"publisher","unstructured":"Ning Wang Bingkun Yao et\u00a0al. 2024. Large Language Model for Verilog Generation with Golden Code Feedback. CoRR abs\/2407.18271 (2024) 9. 10.48550\/ARXIV.2407.18271 arXiv:https:\/\/arXiv.org\/abs\/2407.18271","DOI":"10.48550\/ARXIV.2407.18271"},{"key":"e_1_3_3_1_41_2","unstructured":"Ruiqi Wang Jiyu Guo et\u00a0al. 2025. Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering. arxiv:cs.SE\/2502.06193https:\/\/arxiv.org\/abs\/2502.06193"},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","unstructured":"Ke Xu Jialin Sun et\u00a0al. 2024. MEIC: Re-thinking RTL Debug Automation using LLMs. CoRR abs\/2405.06840 (2024) 11. 10.48550\/ARXIV.2405.06840 arXiv:https:\/\/arXiv.org\/abs\/2405.06840","DOI":"10.48550\/ARXIV.2405.06840"},{"key":"e_1_3_3_1_43_2","doi-asserted-by":"publisher","unstructured":"An Yang Baosong Yang et\u00a0al. 2024. Qwen2.5 Technical Report. CoRR abs\/2412.15115 (2024) 26. 10.48550\/ARXIV.2412.15115 arXiv:https:\/\/arXiv.org\/abs\/2412.15115","DOI":"10.48550\/ARXIV.2412.15115"},{"key":"e_1_3_3_1_44_2","first-page":"125","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, December 10 - 15, 2024","author":"Yang John","year":"2024","unstructured":"John Yang, Carlos\u00a0E. Jimenez, et\u00a0al. 2024. SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering. In Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, December 10 - 15, 2024, Amir Globersons, Lester Mackey, Danielle Belgrave, Angela Fan, Ulrich Paquet, et\u00a0al. (Eds.). Curran Associates, Inc., Vancouver, BC, Canada, 125. http:\/\/papers.nips.cc\/paper_files\/paper\/2024\/hash\/5a7c947568c1b1328ccc5230172e1e7c-Abstract-Conference.html"},{"key":"e_1_3_3_1_45_2","doi-asserted-by":"publisher","unstructured":"Xufeng Yao Yiwen Wang et\u00a0al. 2024. RTLRewriter: Methodologies for Large Models aided RTL Code Optimization. CoRR abs\/2409.11414 (2024) 10. 10.48550\/ARXIV.2409.11414 arXiv:https:\/\/arXiv.org\/abs\/2409.11414","DOI":"10.48550\/ARXIV.2409.11414"},{"key":"e_1_3_3_1_46_2","doi-asserted-by":"publisher","unstructured":"Yongan Zhang Zhongzhi Yu et\u00a0al. 2024. MG-Verilog: Multi-grained Dataset Towards Enhanced LLM-assisted Verilog Generation. CoRR abs\/2407.01910 (2024) 5. 10.48550\/ARXIV.2407.01910 arXiv:https:\/\/arXiv.org\/abs\/2407.01910","DOI":"10.48550\/ARXIV.2407.01910"},{"key":"e_1_3_3_1_47_2","doi-asserted-by":"publisher","unstructured":"Yang Zhao Di Huang et\u00a0al. 2024. CodeV: Empowering LLMs for Verilog Generation through Multi-Level Summarization. CoRR abs\/2407.10424 (2024) 16. 10.48550\/ARXIV.2407.10424 arXiv:https:\/\/arXiv.org\/abs\/2407.10424","DOI":"10.48550\/ARXIV.2407.10424"},{"key":"e_1_3_3_1_48_2","doi-asserted-by":"publisher","unstructured":"Yujie Zhao Hejia Zhang et\u00a0al. 2024. MAGE: A Multi-Agent Engine for Automated RTL Code Generation. CoRR abs\/2412.07822 (2024) 7. 10.48550\/ARXIV.2412.07822 arXiv:https:\/\/arXiv.org\/abs\/2412.07822","DOI":"10.48550\/ARXIV.2412.07822"},{"key":"e_1_3_3_1_49_2","first-page":"29","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, December 10 - 16, 2023","author":"Zheng Lianmin","year":"2023","unstructured":"Lianmin Zheng, Wei-Lin Chiang, et\u00a0al. 2023. Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena. In Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, December 10 - 16, 2023, Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, et\u00a0al. (Eds.). Curran Associates, Inc., New Orleans, LA, USA, 29. http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/91f18a1287b398d378ef22505bf41832-Abstract-Datasets_and_Benchmarks.html"}],"event":{"name":"GLSVLSI '25: Great Lakes Symposium on VLSI 2025","location":"New Orleans LA USA","acronym":"GLSVLSI '25","sponsor":["SIGDA ACM Special Interest Group on Design Automation"]},"container-title":["Proceedings of the Great Lakes Symposium on VLSI 2025"],"original-title":[],"deposited":{"date-parts":[[2025,6,27]],"date-time":"2025-06-27T14:38:23Z","timestamp":1751035103000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3716368.3735171"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,29]]},"references-count":48,"alternative-id":["10.1145\/3716368.3735171","10.1145\/3716368"],"URL":"https:\/\/doi.org\/10.1145\/3716368.3735171","relation":{},"subject":[],"published":{"date-parts":[[2025,6,29]]},"assertion":[{"value":"2025-06-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}