{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,27]],"date-time":"2026-03-27T16:23:38Z","timestamp":1774628618884,"version":"3.50.1"},"reference-count":34,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,10,28]],"date-time":"2023-10-28T00:00:00Z","timestamp":1698451200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,10,28]],"date-time":"2023-10-28T00:00:00Z","timestamp":1698451200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,10,28]]},"DOI":"10.1109\/iccad57390.2023.10323812","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T18:58:45Z","timestamp":1701370725000},"page":"1-8","source":"Crossref","is-referenced-by-count":146,"title":["Invited Paper: VerilogEval: Evaluating Large Language Models for Verilog Code Generation"],"prefix":"10.1109","author":[{"given":"Mingjie","family":"Liu","sequence":"first","affiliation":[{"name":"NVIDIA Corporation"}]},{"given":"Nathaniel","family":"Pinckney","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation"}]},{"given":"Brucek","family":"Khailany","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation"}]},{"given":"Haoxing","family":"Ren","sequence":"additional","affiliation":[{"name":"NVIDIA Corporation"}]}],"member":"263","reference":[{"key":"ref1","first-page":"1877","article-title":"Language mod-els are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref2","volume-title":"Gpt-4 technical report","year":"2023"},{"key":"ref3","article-title":"Sparks of artificial general intelligence: Early experiments with gpt-4","author":"Bubeck","year":"2023","journal-title":"arXiv preprint"},{"key":"ref4","article-title":"Bloomberggpt: A large language model for finance","author":"Wu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.379"},{"key":"ref6","volume-title":"BioMedLM","author":"Bolton"},{"key":"ref7","article-title":"Galactica: A large language model for science","author":"Taylor","year":"2022","journal-title":"arXiv preprint"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00181"},{"key":"ref9","article-title":"Codegen: An open large language model for code with multi-turn program synthesis","author":"Nijkamp","year":"2023","journal-title":"ICLR"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/MLCAD58807.2023.10299874"},{"key":"ref11","article-title":"Chipgpt: How far are we from natural language hardware design","author":"Chang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.23919\/DATE56975.2023.10137086"},{"key":"ref13","article-title":"RTLLM: An open-source benchmark for design rtl generation with large language model","author":"Lu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref14","article-title":"Finetuned language models are zero-shot learners","author":"Wei","year":"2021","journal-title":"arXiv preprint"},{"key":"ref15","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref16","article-title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","author":"Bai","year":"2022","journal-title":"arXiv preprint"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.754"},{"key":"ref18","article-title":"Wizardlm: Empowering large language models to follow complex instructions","author":"Xu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref19","article-title":"WizardCoder: Empowering code large language models with evol-instruct","author":"Luo","year":"2023","journal-title":"arXiv preprint"},{"key":"ref20","article-title":"Textbooks are all you need","author":"Gunasekar","year":"2023","journal-title":"arXiv preprint"},{"key":"ref21","article-title":"Evaluating large language models trained on code.(2021)","author":"Chen","year":"2021","journal-title":"arXiv preprint"},{"key":"ref22","article-title":"Program synthesis with large language models","author":"Austin","year":"2021","journal-title":"arXiv preprint"},{"key":"ref23","article-title":"Measuring coding challenge competence with apps","author":"Hendrycks","year":"2021","journal-title":"arXiv preprint"},{"key":"ref24","volume-title":"The ICARUS verilog compilation system","author":"Williams","year":"2023"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref26","article-title":"CodeXGLUE: A machine learning benchmark dataset for code understanding and generation","volume":"abs\/2102.04664","author":"Lu","year":"2021","journal-title":"CoRR"},{"key":"ref27","article-title":"Spoc: Search-based pseudocode to code","volume":"32","author":"Kulal","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-16214-0_42"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/SEQUEN.1997.666900"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.577"},{"key":"ref31","article-title":"The curious case of neural text degeneration","author":"Holtzman","year":"2019","journal-title":"arXiv preprint"},{"key":"ref32","article-title":"The pile: An 800gb dataset of diverse text for language modeling","author":"Gao","year":"2020","journal-title":"arXiv preprint"},{"key":"ref33","volume-title":"OpenAI models api","year":"2023"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623316"}],"event":{"name":"2023 IEEE\/ACM International Conference on Computer Aided Design (ICCAD)","location":"San Francisco, CA, USA","start":{"date-parts":[[2023,10,28]]},"end":{"date-parts":[[2023,11,2]]}},"container-title":["2023 IEEE\/ACM International Conference on Computer Aided Design (ICCAD)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10323590\/10323543\/10323812.pdf?arnumber=10323812","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,2]],"date-time":"2024-03-02T22:10:44Z","timestamp":1709417444000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10323812\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,28]]},"references-count":34,"URL":"https:\/\/doi.org\/10.1109\/iccad57390.2023.10323812","relation":{},"subject":[],"published":{"date-parts":[[2023,10,28]]}}}