{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,22]],"date-time":"2025-10-22T18:27:38Z","timestamp":1761157658799,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T00:00:00Z","timestamp":1725840000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,9]]},"DOI":"10.1145\/3670474.3685948","type":"proceedings-article","created":{"date-parts":[[2024,9,3]],"date-time":"2024-09-03T06:22:27Z","timestamp":1725344547000},"page":"1-17","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["PyHDL-Eval: An LLM Evaluation Framework for Hardware Design Using Python-Embedded DSLs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2835-667X","authenticated-orcid":false,"given":"Christopher","family":"Batten","sequence":"first","affiliation":[{"name":"Cornell University, Ithaca, NY, USA and NVIDIA Corporation"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6159-8964","authenticated-orcid":false,"given":"Nathaniel","family":"Pinckney","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation, Santa Clara, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3488-9763","authenticated-orcid":false,"given":"Mingjie","family":"Liu","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation, Santa Clara, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1028-3860","authenticated-orcid":false,"given":"Haoxing","family":"Ren","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation, Santa Clara, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7584-3489","authenticated-orcid":false,"given":"Brucek","family":"Khailany","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation, Santa Clara, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,9,9]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"May","author":"Allam A.","year":"2024","unstructured":"A. Allam and M. Shalan. RTL-Repo: A Benchmark for Evaluating LLMs on Large-Scale RTL Design Projects. Computing Research Repository (CoRR), arXiv:2405.17378, May 2024."},{"key":"e_1_3_2_1_2_1","unstructured":"Amaranth HDL. Online Webpage 2024 (accessed May 2024). https:\/\/github.com\/amaranth-lang\/amaranth."},{"key":"e_1_3_2_1_3_1","volume-title":"C\u03bblash: Structural Descriptions of Synchronous Hardware Using Haskell. Euromicro Conf. on Digital System Design (DSD)","author":"Baaij C.","year":"2010","unstructured":"C. Baaij, M. Kooijman, J. Kuper, A. Boeijink, and M. Gerards. C\u03bblash: Structural Descriptions of Synchronous Hardware Using Haskell. Euromicro Conf. on Digital System Design (DSD), Sep 2010."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2228360.2228584"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.21236\/ADA450477"},{"key":"e_1_3_2_1_6_1","volume-title":"Sep","author":"Bjesse P.","year":"1998","unstructured":"P. Bjesse, K. Claessen, M. Sheeran, and S. Singh. Lava: Hardware Design in Haskell. Int'l Conf. on Functional Programming (ICFP), Sep 1998."},{"key":"e_1_3_2_1_7_1","volume-title":"Sep","author":"Blocklove J.","year":"2023","unstructured":"J. Blocklove, S. Garg, R. Karri, and H. Pearce. Chip-Chat: Challenges and Opportunities in Conversational Hardware Design. Int'l Symp. on Machine Learning for CAD (MLCAD), Sep 2023."},{"key":"e_1_3_2_1_8_1","volume-title":"Conf. on Neural Information Processing Systems (NeurIPS)","author":"Brown T. B.","year":"2020","unstructured":"T. B. Brown, B. Mann, N. Ryder, M. Subbiah, J. Kaplan, P. Dhariwal, A. Neelakantan, P. Shyam, G. Sastry, A. Askell, S. Agarwal, A. Herbert-Voss, G. Krueger, T. Henighan, R. Child, A. Ramesh, D. M. Ziegler, J. Wu, C. Winter, C. Hesse, M. Chen, E. Sigler, M. Litwin, S. Gray, B. Chess, J. Clark, C. Berner, S. McCandlish, A. Radford, I. Sutskever, and D. Amodei. Language Modles are Few-Shot Learners. Conf. on Neural Information Processing Systems (NeurIPS), Dec 2020."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3649329.3657356"},{"key":"e_1_3_2_1_10_1","volume-title":"May","author":"Chang K.","year":"2023","unstructured":"K. Chang, Y. Wang, H. Ren, M. Wang, S. Liang, Y. Han, H. Li, and X. Li. ChipGPT: How Far Are We From Natural Language Hardware Design? Computing Research Repository (CoRR), arXiv:2305.14019, May 2023."},{"key":"e_1_3_2_1_11_1","volume-title":"Sep","author":"Clow J.","year":"2017","unstructured":"J. Clow, G. Tzimpragos, D. Dangwal, S. Guo, J. McMahan, and T. Sherwood. A Pythonic Approach for Rapid Hardware Prototyping and Instrumentation. Int'l Conf. on Field Programmable Logic (FPL), Sep 2017."},{"volume-title":"Online Webpage, 2024","year":"2024","key":"e_1_3_2_1_12_1","unstructured":"cocotb: A Coroutine-Based Cosimulation Library for Writing VHDL and Verilog Testbenches in Python. Online Webpage, 2024 (accessed May 2024). https: \/\/github.com\/cocotb\/cocotb."},{"key":"e_1_3_2_1_13_1","article-title":"A Python-Based Hardware Description Language","author":"Decaluwe J.","year":"2004","unstructured":"J.Decaluwe. MyHDL: A Python-Based Hardware Description Language. Linux Journal, Nov 2004.","journal-title":"Linux Journal"},{"key":"e_1_3_2_1_14_1","volume-title":"Type-Directed Scheduling of Streaming Accelerators. Conf. on Programming Language Design and Implementation (PLDI)","author":"Durst D.","year":"2020","unstructured":"D. Durst, M. Feldman, D. Huff, D. Akeley, R. Daly, G. L. Bernstein, M. Patrignani, K. Fatahalian, and P. Hanrahan. Type-Directed Scheduling of Streaming Accelerators. Conf. on Programming Language Design and Implementation (PLDI), Jun 2020."},{"key":"e_1_3_2_1_15_1","volume-title":"Nov","author":"Fu Y.","year":"2023","unstructured":"Y. Fu, Y. Zhang, Z. Yu, S. Li, Z. Ye, C. Li, C. Wan, and Y. C. Lin. GPT4AIChip: Towards Next-Generation AI Accelerator Design Automation via Large Language Models. Int'l Conf. on Computer-Aided Design (ICCAD), Nov 2023."},{"volume-title":"Google White Paper","year":"2024","key":"e_1_3_2_1_16_1","unstructured":"Google. CodeGemma: Open Code Models Based on Gemma. Google White Paper, May 2024. https:\/\/goo.gle\/codegemma."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/MDAT.2020.3024144"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2020.2997638"},{"key":"e_1_3_2_1_19_1","volume-title":"LiteX: An Open-Source SoC Builder and Library Based on Migen Python DSL. Workshop on Open-Source Design Automation (OSDA)","author":"Kermarrec F.","year":"2019","unstructured":"F. Kermarrec, S. Bourdeauducq, J.-C. L. Lann, and H. Badier. LiteX: An Open-Source SoC Builder and Library Based on Migen Python DSL. Workshop on Open-Source Design Automation (OSDA), Mar 2019."},{"key":"e_1_3_2_1_20_1","volume-title":"Oct","author":"Liu M.","year":"2023","unstructured":"M. Liu, T.-D. Ene, R. Kirby, C. Cheng, N. Pinckney, R. Liang, J. Alben, H. Anand, S. Banerjee, I. Bayraktaroglu, B. Bhaskaran, B. Catanzaro, A. Chaudhuri, S. Clay, B. Dally, L. Dang, P. Deshpande, S. Dhodhi, S. Halepete, E. Hill, J. Hu, S. Jain, A. Jindal, B. Khailany, G. Kokai, K. Kunal, X. Li, C. Lind, H. Liu, S. Oberman, S. Omar, G. Pasandi, S. Pratty, J. Raiman, A. Sarkar, Z. Shao, H. Sun, P. P. Suthar, V. Tej, W. Turner, K. Xu, and H. Ren. ChipNeMo: Domain-Adapted LLMs for Chip Design. Computing Research Repository (CoRR), arXiv:2311.00176, Oct 2023."},{"key":"e_1_3_2_1_21_1","volume-title":"Nov","author":"Liu M.","year":"2023","unstructured":"M. Liu, N. Pinckney, B. Khailany, and H. Ren. VerilogEval: Evaluating Large Language Models for Verilog Code Generation. Int'l Conf. on Computer-Aided Design (ICCAD), Nov 2023."},{"key":"e_1_3_2_1_22_1","volume-title":"Dec","author":"Liu S.","year":"2023","unstructured":"S. Liu, W. Fang, Y. Lu, Q. Zhang, H. Zhang, and Z. Xie. RTLCoder: Outperforming GPT-3.5 in Design RTL Generation with Our Open-Source Dataset and Lightweight Solution. Computing Research Repository (CoRR), arXiv:2312.08617, Dec 2023."},{"key":"e_1_3_2_1_23_1","volume-title":"Dec","author":"Lockhart D.","year":"2014","unstructured":"D. Lockhart, G. Zibrat, and C. Batten. PyMTL: A Unified Framework for Vertically Integrated Computer Architecture Research. Int'l Symp. on Microarchitecture (MICRO), Dec 2014."},{"key":"e_1_3_2_1_24_1","volume-title":"RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Models. Asia and South Pacific Design Automation Conf. (ASP-DAC)","author":"Lu Y.","year":"2024","unstructured":"Y. Lu, S. Liu, Q. Zhang, and Z. Xie. RTLLM: An Open-Source Benchmark for Design RTL Generation with Large Language Models. Asia and South Pacific Design Automation Conf. (ASP-DAC), Jan 2024."},{"key":"e_1_3_2_1_25_1","volume-title":"Nov","author":"MacIver D. R.","year":"2019","unstructured":"D. R. MacIver, Z. Hatfield-Dodds, and many other contributors. Hypothesis: A New Approach to Property-Based Testing. Journal of Open-Source Software (JOSS), 4(43), Nov 2019."},{"volume-title":"Online Webpage","year":"2024","key":"e_1_3_2_1_26_1","unstructured":"Meta. Introducing Meta Llama 3: The Most Capable Openly Available LLM to Date. Online Webpage, Apr 2024 (accessed May 2024). https:\/\/ai.meta.com\/blog\/meta-llama-3."},{"key":"e_1_3_2_1_27_1","volume-title":"Modular Hardawre Design with Timeline Types. Conf. on Programming Language Design and Implementation (PLDI)","author":"Nigam R.","year":"2023","unstructured":"R. Nigam, P. H. A. de Amorim, and A. Sampson. Modular Hardawre Design with Timeline Types. Conf. on Programming Language Design and Implementation (PLDI), Jun 2023."},{"key":"e_1_3_2_1_28_1","volume-title":"Correct RTL from High-Level Specifications. Int'l Conf. on Formal Methods and Models for Co-Design (MEMOCODE)","author":"Nikhil N.","year":"2004","unstructured":"N. Nikhil. Bluespec System Verilog: Efficient, Correct RTL from High-Level Specifications. Int'l Conf. on Formal Methods and Models for Co-Design (MEMOCODE), Jun 2004."},{"key":"e_1_3_2_1_29_1","volume-title":"Online Webpage","author":"AI.","year":"2024","unstructured":"OpenAI. New Models and Developer Products Announced at DevDay. Online Webpage, Nov 2024 (accessed May 2024). https:\/\/openai.com\/index\/new-models-and-developer-products-announced-at-devday."},{"key":"e_1_3_2_1_30_1","unstructured":"OpenAI et al. GPT-4 Technical Report. Computing Research Repository (CoRR) arxiv:2303.08774 Mar 2023."},{"key":"e_1_3_2_1_31_1","volume-title":"Sep","author":"Orenes-Vera M.","year":"2023","unstructured":"M. Orenes-Vera, M. Martonosi, and D. Wentzlaff. Using LLMs to Facilitate Formal Verification of RTL. Computing Research Repository (CoRR), arXiv:2309.09437, Sep 2023."},{"key":"e_1_3_2_1_32_1","volume-title":"Sep","author":"Port O.","year":"2017","unstructured":"O. Port and Y. Etsion. DFiant: A Dataflow Hardware Description Language. Int'l Conf. on Field Programmable Logic (FPL), Sep 2017."},{"key":"e_1_3_2_1_33_1","volume-title":"Dec","author":"Ray A.","year":"2023","unstructured":"A. Ray, B. Devlin, F. Y. Quah, and R. Yesantharao. HardCaml: An OCaml Hardware Domain-Specific Languaeg for Efficient and Robust Design. Computing Research Repository (CoRR), arXiv:1509.02058, Dec 2023."},{"volume-title":"Online Webpage, 2024","year":"2024","key":"e_1_3_2_1_34_1","unstructured":"SpinalHDL: A Scala-based HDL. Online Webpage, 2024 (accessed May 2024). https:\/\/github.com\/SpinalHDL\/SpinalHDL."},{"key":"e_1_3_2_1_35_1","volume-title":"Apr","author":"Thakur S.","year":"2023","unstructured":"S. Thakur, B. Ahmad, Z. Fan, H. Pearce, B. Tan, R. Karri, B. Dolan-Gavitt, and S. Garg. Benchmarking Large Language Models for Automated Verilog RTL Code Generation. Design, Automation, and Test in Europe (DATE), Apr 2023."},{"key":"e_1_3_2_1_36_1","volume-title":"Apr","author":"Thakur S.","year":"2024","unstructured":"S. Thakur, B. Ahmad, H. Pearce, B. Tan, B. Dolan-Gavitt, R. Karri, and S. Garg. VeriGen: A Large Language Model for Verilog Code Generation. ACM Trans. on Design Automation of Electronic Systems (TODAES), 29(3):1--31, Apr 2024."},{"key":"e_1_3_2_1_37_1","volume-title":"Nov","author":"Thakur S.","year":"2023","unstructured":"S. Thakur, J. Blocklove, H. Pearce, B. Tan, S. Garg, and R. Karri. AutoChip: Automating HDL Generation Using LLM Feedback. Computing Research Repository (CoRR), arXiv:2311.04887, Nov 2023."},{"key":"e_1_3_2_1_38_1","volume-title":"Summit on Advances in Programming Languages (SNAPL)","author":"Truong L.","year":"2019","unstructured":"L. Truong and P. Hanrahan. A Golden Age of Hardware Description Languages: Applying Programming Language Techniques to Improve Design Productivity. Summit on Advances in Programming Languages (SNAPL), May 2019."},{"key":"e_1_3_2_1_39_1","volume-title":"Nov","author":"Tsai Y.-D.","year":"2023","unstructured":"Y.-D. Tsai, M. Liu, and H. Ren. RTLFixer: Automatically Fixing RTL Syntax Errors with Large Language Models. Computing Research Repository (CoRR), arXivv:2311.16543, Nov 2023."},{"key":"e_1_3_2_1_40_1","volume-title":"Jun","author":"Zakharov M.","year":"2024","unstructured":"M. Zakharov, F. R. Kashanaki, and J. Renau. HDLEval Benchmarking LLMs for Multiple HDLs. Int'l Workshop on LLM-Aided Design (LAD), Jun 2024."},{"key":"e_1_3_2_1_41_1","volume-title":"Oct","author":"Zhang Z.","year":"2023","unstructured":"Z. Zhang, G. Chadwick, H. McNally, Y. Zhao, and R. Mullins. LLM4DV: Using Large Language Models for Hardware Test Stimuli Generation. Computing Research Repository (CoRR), arXiv:2310.04535, Oct 2023."},{"key":"e_1_3_2_1_42_1","volume-title":"Dec","author":"Zhong R.","year":"2023","unstructured":"R. Zhong, X. Du, S. Kai, Z. Tang, S. Xu, H.-L. Zhen, J. Hao, Q. Xu, M. Yuan, and J. Yan. LLM4EDA: Emerging Progress in Large Language Models for Electronic Design Automation. Computing Research Repository (CoRR), arXiv:2401.12224, Dec 2023."}],"event":{"name":"MLCAD '24: 2024 ACM\/IEEE International Symposium on Machine Learning for CAD","sponsor":["SIGDA ACM Special Interest Group on Design Automation","IEEE CEDA"],"location":"Salt Lake City UT USA","acronym":"MLCAD '24"},"container-title":["Proceedings of the 2024 ACM\/IEEE International Symposium on Machine Learning for CAD"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3670474.3685948","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3670474.3685948","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:44:32Z","timestamp":1755906272000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3670474.3685948"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,9]]},"references-count":42,"alternative-id":["10.1145\/3670474.3685948","10.1145\/3670474"],"URL":"https:\/\/doi.org\/10.1145\/3670474.3685948","relation":{},"subject":[],"published":{"date-parts":[[2024,9,9]]},"assertion":[{"value":"2024-09-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}