{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T16:40:00Z","timestamp":1781887200235,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":21,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"NSFC","award":["62090024"],"award-info":[{"award-number":["62090024"]}]},{"name":"NSFC","award":["62222411"],"award-info":[{"award-number":["62222411"]}]},{"name":"NSFC","award":["62025404"],"award-info":[{"award-number":["62025404"]}]},{"name":"NSFC","award":["92373206"],"award-info":[{"award-number":["92373206"]}]},{"name":"NSFC","award":["62202453"],"award-info":[{"award-number":["62202453"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3676536.3676679","type":"proceedings-article","created":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T12:53:56Z","timestamp":1744203236000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":11,"title":["Natural language is not enough: Benchmarking multi-modal generative AI for Verilog generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1920-0101","authenticated-orcid":false,"given":"Kaiyan","family":"Chang","sequence":"first","affiliation":[{"name":"State Key Lab of Processors, Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"},{"name":"University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7039-7707","authenticated-orcid":false,"given":"Zhirong","family":"Chen","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5158-7417","authenticated-orcid":false,"given":"Yunhao","family":"Zhou","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-9237-559X","authenticated-orcid":false,"given":"Wenlong","family":"Zhu","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-3273-2271","authenticated-orcid":false,"given":"Kun","family":"Wang","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"},{"name":"University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0243-6516","authenticated-orcid":false,"given":"Haobo","family":"Xu","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7193-464X","authenticated-orcid":false,"given":"Cangyuan","family":"Li","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7012-2308","authenticated-orcid":false,"given":"Mengdi","family":"Wang","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8407-2594","authenticated-orcid":false,"given":"Shengwen","family":"Liang","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8082-4218","authenticated-orcid":false,"given":"Huawei","family":"Li","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"},{"name":"University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5113-8067","authenticated-orcid":false,"given":"Yinhe","family":"Han","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5172-4736","authenticated-orcid":false,"given":"Ying","family":"Wang","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,9]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Fixing hardware security bugs with large language models. arXiv preprint arXiv:2302.01215","author":"Ahmad B.","year":"2023","unstructured":"Ahmad, B., Thakur, S., Tan, B., Karri, R., and Pearce, H. Fixing hardware security bugs with large language models. arXiv preprint arXiv:2302.01215 (2023)."},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of the 35th Annual ACM Symposium on User Interface Software and Technology (New York, NY, USA, 2022), UIST '22, Association for Computing Machinery.","author":"Arawjo I.","unstructured":"Arawjo, I., DeArmas, A., Roberts, M., Basu, S., and Parikh, T. Notational programming for notebook environments: A case study with quantum circuits. In Proceedings of the 35th Annual ACM Symposium on User Interface Software and Technology (New York, NY, USA, 2022), UIST '22, Association for Computing Machinery."},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of the ACM on Programming Languages 7, PLDI (2023)","author":"Beurer-Kellner L.","unstructured":"Beurer-Kellner, L., Fischer, M., and Vechev, M. Prompting is programming: A query language for large language models. Proceedings of the ACM on Programming Languages 7, PLDI (2023), 1946--1969."},{"key":"e_1_3_2_1_4_1","volume-title":"Chip-chat: Challenges and opportunities in conversational hardware design. arXiv preprint arXiv:2305.13243","author":"Blocklove J.","year":"2023","unstructured":"Blocklove, J., Garg, S., Karri, R., and Pearce, H. Chip-chat: Challenges and opportunities in conversational hardware design. arXiv preprint arXiv:2305.13243 (2023)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3649329.3657356"},{"key":"e_1_3_2_1_6_1","volume-title":"Chipgpt: How far are we from natural language hardware design. arXiv preprint arXiv:2305.14019","author":"Chang K.","year":"2023","unstructured":"Chang, K., Wang, Y., Ren, H., Wang, M., Liang, S., Han, Y., Li, H., and Li, X. Chipgpt: How far are we from natural language hardware design. arXiv preprint arXiv:2305.14019 (2023)."},{"key":"e_1_3_2_1_7_1","volume-title":"The dawn of ai-native eda: Promises and challenges of large circuit models. arXiv preprint arXiv:2403.07257","author":"Chen L.","year":"2024","unstructured":"Chen, L., Chen, Y., Chu, Z., Fang, W., Ho, T.-Y., Huang, Y., Khan, S., Li, M., Li, X., Liang, Y., et al. The dawn of ai-native eda: Promises and challenges of large circuit models. arXiv preprint arXiv:2403.07257 (2024)."},{"key":"e_1_3_2_1_8_1","volume-title":"Assertllm: Generating and evaluating hardware verification assertions from design specifications via multi-llms. arXiv preprint arXiv:2402.00386","author":"Fang W.","year":"2024","unstructured":"Fang, W., Li, M., Li, M., Yan, Z., Liu, S., Zhang, H., and Xie, Z. Assertllm: Generating and evaluating hardware verification assertions from design specifications via multi-llms. arXiv preprint arXiv:2402.00386 (2024)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCAD57390.2023.10323953"},{"key":"e_1_3_2_1_10_1","volume-title":"Chateda: A large language model powered autonomous agent for eda. arXiv preprint arXiv:2308.10204","author":"He Z.","year":"2023","unstructured":"He, Z., Wu, H., Zhang, X., Yao, X., Zheng, S., Zheng, H., and Yu, B. Chateda: A large language model powered autonomous agent for eda. arXiv preprint arXiv:2308.10204 (2023)."},{"key":"e_1_3_2_1_11_1","volume-title":"Llm-assisted generation of hardware assertions. arXiv preprint arXiv:2306.14027","author":"Kande R.","year":"2023","unstructured":"Kande, R., Pearce, H., Tan, B., Dolan-Gavitt, B., Thakur, S., Karri, R., and Rajendran, J. Llm-assisted generation of hardware assertions. arXiv preprint arXiv:2306.14027 (2023)."},{"key":"e_1_3_2_1_12_1","volume-title":"Unleashing the potential of llms for quantum computing: A study in quantum architecture design. arXiv preprint arXiv:2307.08191","author":"Liang Z.","year":"2023","unstructured":"Liang, Z., Cheng, J., Yang, R., Ren, H., Song, Z., Wu, D., Qian, X., Li, T., and Shi, Y. Unleashing the potential of llms for quantum computing: A study in quantum architecture design. arXiv preprint arXiv:2307.08191 (2023)."},{"key":"e_1_3_2_1_13_1","volume-title":"2023 IEEE\/ACM International Conference on Computer-Aided Design (ICCAD)","author":"Liu M.","year":"2023","unstructured":"Liu, M., Pinckney, N., Khailany, B., and Ren, H. VerilogEval: evaluating large language models for verilog code generation. In 2023 IEEE\/ACM International Conference on Computer-Aided Design (ICCAD) (2023)."},{"key":"e_1_3_2_1_14_1","volume-title":"Rtllm: An open-source benchmark for design rtl generation with large language model","author":"Lu Y.","year":"2023","unstructured":"Lu, Y., Liu, S., Zhang, Q., and Xie, Z. Rtllm: An open-source benchmark for design rtl generation with large language model, 2023."},{"key":"e_1_3_2_1_15_1","volume-title":"Chipnemo: Domain-adapted llms for chip design. arXiv preprint arXiv:2307.09288","author":"Mingjie Liu\u00a7 T. E.","year":"2023","unstructured":"Mingjie Liu\u00a7, T. E. Chipnemo: Domain-adapted llms for chip design. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_16_1","volume-title":"From rtl to sva: Llm-assisted generation of formal verification testbenches","author":"Orenes-Vera M.","year":"2023","unstructured":"Orenes-Vera, M., Martonosi, M., and Wentzlaff, D. From rtl to sva: Llm-assisted generation of formal verification testbenches, 2023."},{"key":"e_1_3_2_1_17_1","first-page":"1","volume-title":"Automation & Test in Europe Conference & Exhibition (DATE)","author":"Thakur S.","year":"2023","unstructured":"Thakur, S., Ahmad, B., Fan, Z., Pearce, H., Tan, B., Karri, R., Dolan-Gavitt, B., and Garg, S. Benchmarking large language models for automated verilog rtl code generation. In 2023 Design, Automation & Test in Europe Conference & Exhibition (DATE) (2023), IEEE, pp. 1--6."},{"key":"e_1_3_2_1_18_1","volume-title":"Verigen: A large language model for verilog code generation. arXiv preprint arXiv:2308.00708","author":"Thakur S.","year":"2023","unstructured":"Thakur, S., Ahmad, B., Pearce, H., Tan, B., Dolan-Gavitt, B., Karri, R., and Garg, S. Verigen: A large language model for verilog code generation. arXiv preprint arXiv:2308.00708 (2023)."},{"key":"e_1_3_2_1_19_1","volume-title":"Rtlfixer: Automatically fixing rtl syntax errors with large language models. arXiv preprint arXiv:2311.16543","author":"Tsai Y.","year":"2023","unstructured":"Tsai, Y., Liu, M., and Ren, H. Rtlfixer: Automatically fixing rtl syntax errors with large language models. arXiv preprint arXiv:2311.16543 (2023)."},{"key":"e_1_3_2_1_20_1","volume-title":"On the viability of using llms for sw\/hw co-design: An example in designing cim dnn accelerators. arXiv preprint arXiv:2306.06923","author":"Yan Z.","year":"2023","unstructured":"Yan, Z., Qin, Y., Hu, X. S., and Shi, Y. On the viability of using llms for sw\/hw co-design: An example in designing cim dnn accelerators. arXiv preprint arXiv:2306.06923 (2023)."},{"key":"e_1_3_2_1_21_1","volume-title":"Hdldebugger: Streamlining hdl debugging with large language models. arXiv preprint arXiv:2403.11671","author":"Yao X.","year":"2024","unstructured":"Yao, X., Li, H., Chan, T. H., Xiao, W., Yuan, M., Huang, Y., Chen, L., and Yu, B. Hdldebugger: Streamlining hdl debugging with large language models. arXiv preprint arXiv:2403.11671 (2024)."}],"event":{"name":"ICCAD '24: 43rd IEEE\/ACM International Conference on Computer-Aided Design","location":"Newark Liberty International Airport Marriott New York NY USA","acronym":"ICCAD '24","sponsor":["SIGDA ACM Special Interest Group on Design Automation","IEEE CAS","IEEE CEDA","IEEE EDS"]},"container-title":["Proceedings of the 43rd IEEE\/ACM International Conference on Computer-Aided Design"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676536.3676679","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3676536.3676679","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:43:57Z","timestamp":1750290237000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676536.3676679"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":21,"alternative-id":["10.1145\/3676536.3676679","10.1145\/3676536"],"URL":"https:\/\/doi.org\/10.1145\/3676536.3676679","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2025-04-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}