{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T20:50:26Z","timestamp":1775249426413,"version":"3.50.1"},"reference-count":54,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,9,8]],"date-time":"2025-09-08T00:00:00Z","timestamp":1757289600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,9,8]],"date-time":"2025-09-08T00:00:00Z","timestamp":1757289600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100002809","name":"Generalitat de Catalunya","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,9,8]]},"DOI":"10.1109\/mlcad65511.2025.11189228","type":"proceedings-article","created":{"date-parts":[[2025,10,9]],"date-time":"2025-10-09T17:51:22Z","timestamp":1760032282000},"page":"1-12","source":"Crossref","is-referenced-by-count":1,"title":["TuRTLe: A Unified Evaluation of LLMs for RTL Generation"],"prefix":"10.1109","author":[{"given":"Dario","family":"Garcia-Gasulla","sequence":"first","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Gokcen","family":"Kestor","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Emanuele","family":"Parisi","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Miquel","family":"Albert\u00ed-Binimelis","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Cristian","family":"Gutierrez","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Razine Moundir","family":"Ghorab","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Orlando","family":"Montenegro","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Bernat","family":"Homs","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]},{"given":"Miquel","family":"Moreto","sequence":"additional","affiliation":[{"name":"Barcelona Supercomputing Center (BSC)"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Language models are few-shot learners","volume-title":"Proceedings of the 34th International Conference on Neural Information Processing Systems, ser. NIPS \u201920","author":"Brown"},{"key":"ref2","article-title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","author":"Bubeck","year":"2023"},{"key":"ref3","article-title":"BloombergGPT: A Large Language Model for Finance","author":"Wu","year":"2023"},{"key":"ref4","first-page":"4700","article-title":"BioMegatron: Larger Biomedical Domain Language Model","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Shin"},{"key":"ref5","article-title":"Galactica: A Large Language Model for Science","author":"Taylor","year":"2022"},{"key":"ref6","article-title":"Exploring the Benefits of Domain-Pretraining of Generative Large Language Models for Chemistry","author":"Acharya","year":"2024"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/icse48619.2023.00181"},{"key":"ref8","article-title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","volume-title":"The Eleventh International Conference on Learning Representations","author":"Nijkamp"},{"key":"ref9","article-title":"Code Llama: Open Foundation Models for Code","author":"Rozi\u00e8re","year":"2024"},{"key":"ref10","article-title":"Model card for starchat2 15b","author":"team","year":"2024"},{"key":"ref11","article-title":"The Dawn of AI-Native EDA: Opportunities and Challenges of Large Circuit Models","author":"Chen","year":"2024"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3715324"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3670474.3685964"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/MLCAD58807.2023.10299874"},{"key":"ref15","article-title":"ChipGPT: How far are we from natural language hardware design","author":"Chang","year":"2023"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3643681"},{"key":"ref17","article-title":"ChipNeMo: Domain-Adapted LLMs for Chip Design","author":"Liu","year":"2024"},{"key":"ref18","article-title":"RTLCoder: Fully Open-Source and Efficient LLM-Assisted RTL Code Generation Technique","author":"Liu","year":"2024"},{"key":"ref19","article-title":"BetterV: controlled verilog generation with discriminative guidance","volume-title":"Proceedings of the 41st International Conference on Machine Learning, ser. ICML\u201924","author":"Pei"},{"key":"ref20","article-title":"A Deep Learning Framework for Verilog Autocompletion Towards Design and Verification Automation","author":"Dehaerne","year":"2023"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3649329.3657353"},{"key":"ref22","article-title":"Towards LLM-Powered Verilog RTL Assistant: Self-Verification and Self-Correction","author":"Huang","year":"2024"},{"key":"ref23","article-title":"AIvril: AI-Driven RTL Generation With Verification In-The-Loop","author":"ul Islam","year":"2024"},{"key":"ref24","article-title":"MAGE: A Multi-Agent Engine for Automated RTL Code Generation","author":"Zhao","year":"2024"},{"key":"ref25","article-title":"PromptV: Leveraging LLM-powered Multi-Agent Prompting for High-quality Verilog Generation","author":"Mi","year":"2024"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICCAD57390.2023.10323812"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3718088"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/LAD62341.2024.10691810"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/ASP-DAC58780.2024.10473904"},{"key":"ref30","article-title":"A framework for the evaluation of code generation models","author":"Ben Allal","year":"2022"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3649329.3657353"},{"key":"ref32","first-page":"21 558","article-title":"Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation","volume":"36","author":"Liu","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.449"},{"key":"ref34","article-title":"Advanced Large Language Model (LLM)-Driven Verilog Development: Enhancing Power, Performance, and Area Optimization in Code Synthesis","author":"Thorat","year":"2023"},{"key":"ref35","first-page":"1","article-title":"Building OpenLANE: A 130nm OpenROAD-based Tapeout-Proven Flow","volume-title":"2020 IEEE\/ACM International Conference On Computer Aided Design (ICCAD)","author":"Shalan"},{"key":"ref36","article-title":"The ICARUS Verilog Compilation System","author":"Williams","year":"2023"},{"key":"ref37","article-title":"Spoc: Search-based pseudocode to code","volume":"32","author":"Kulal","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref38","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref39","article-title":"Google\/SkyWater and the Promise of the Open PDK","volume-title":"Workshop on Open-Source EDA Technology","author":"Edwards"},{"key":"ref40","article-title":"Llama: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.1591"},{"key":"ref42","doi-asserted-by":"crossref","DOI":"10.1109\/TCAD.2025.3604320","article-title":"Codev: Empowering llms with hdl generation through multi-level summarization","author":"Zhao","year":"2025"},{"key":"ref43","article-title":"Qwen2.5 technical report","author":"Yang","year":"2025"},{"key":"ref44","article-title":"Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning","year":"2025"},{"key":"ref45","article-title":"Qwen3 technical report","author":"Team","year":"2025"},{"key":"ref46","article-title":"Seed-Coder: Let the Code Model Curate Data for Itself","author":"Seed","year":"2024"},{"key":"ref47","article-title":"Qwen2.5-coder technical report","author":"Hui","year":"2024"},{"key":"ref48","article-title":"Deepseek-coder: When the large language model meets programming \u2013 the rise of code intelligence","author":"Guo","year":"2024"},{"key":"ref49","article-title":"The llama 3 herd of models","author":"Grattafiori","year":"2024"},{"key":"ref50","article-title":"Qwq-32b: Embracing the power of reinforcement learning","year":"2025"},{"key":"ref51","article-title":"Deepcoder: A fully open-source 14b coder at o3-mini level","author":"Luo","year":"2025"},{"key":"ref52","article-title":"Codev model distilled from deepseek-r1","year":"2025"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3676536.3676830"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.23919\/DATE64628.2025.10993072"}],"event":{"name":"2025 ACM\/IEEE 7th Symposium on Machine Learning for CAD (MLCAD)","location":"Santa Cruz, CA, USA","start":{"date-parts":[[2025,9,8]]},"end":{"date-parts":[[2025,9,10]]}},"container-title":["2025 ACM\/IEEE 7th Symposium on Machine Learning for CAD (MLCAD)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11189084\/11189054\/11189228.pdf?arnumber=11189228","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T19:50:02Z","timestamp":1775245802000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11189228\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,8]]},"references-count":54,"URL":"https:\/\/doi.org\/10.1109\/mlcad65511.2025.11189228","relation":{},"subject":[],"published":{"date-parts":[[2025,9,8]]}}}