{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T03:08:03Z","timestamp":1784257683418,"version":"3.55.0"},"reference-count":49,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,10,26]],"date-time":"2025-10-26T00:00:00Z","timestamp":1761436800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,26]],"date-time":"2025-10-26T00:00:00Z","timestamp":1761436800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100003816","name":"Huawei Technologies","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003816","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,10,26]]},"DOI":"10.1109\/iccad66269.2025.11240653","type":"proceedings-article","created":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T18:39:34Z","timestamp":1763663974000},"page":"1-9","source":"Crossref","is-referenced-by-count":1,"title":["MMCircuitEval: A Comprehensive Multimodal Circuit-Focused Benchmark for Evaluating LLMs"],"prefix":"10.1109","author":[{"given":"Chenchen","family":"Zhao","sequence":"first","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhengyuan","family":"Shi","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiangyu","family":"Wen","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chengjie","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanjing University,School of Electronic Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yi","family":"Liu","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yunhao","family":"Zhou","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuxiang","family":"Zhao","sequence":"additional","affiliation":[{"name":"Peking University,School of Integrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hefei","family":"Feng","sequence":"additional","affiliation":[{"name":"Southeast University,School of Intergrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yinan","family":"Zhu","sequence":"additional","affiliation":[{"name":"National Center of Technology Innovation for EDA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gwok-Waa","family":"Wan","sequence":"additional","affiliation":[{"name":"National Center of Technology Innovation for EDA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xin","family":"Cheng","sequence":"additional","affiliation":[{"name":"Southeast University,School of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanjing University,School of Electronic Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yongqi","family":"Fu","sequence":"additional","affiliation":[{"name":"Southeast University,School of Intergrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chujie","family":"Chen","sequence":"additional","affiliation":[{"name":"University of Chinese Academy of Sciences,Department of Computer Science and Technology"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chenhao","family":"Xue","sequence":"additional","affiliation":[{"name":"Peking University,School of Integrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ying","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Chinese Academy of Sciences,Department of Computer Science and Technology"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yibo","family":"Lin","sequence":"additional","affiliation":[{"name":"Peking University,School of Integrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jun","family":"Yang","sequence":"additional","affiliation":[{"name":"Southeast University,School of Intergrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ning","family":"Xu","sequence":"additional","affiliation":[{"name":"Southeast University,School of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xi","family":"Wang","sequence":"additional","affiliation":[{"name":"Southeast University,School of Intergrated Circuits"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qiang","family":"Xu","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong,Department of Computer Science and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Chipnemo: Domain-adapted llms for chip design","author":"Liu","year":"2023"},{"key":"ref2","article-title":"Semikong: Curating, training, and evaluating a semiconductor industry-specific large language model","author":"Nguyen","year":"2024"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3676536.3697118"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ASP-DAC58780.2024.10473904"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/MLCAD62225.2024.10740250"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICCAD57390.2023.10323812"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/LAD62341.2024.10691774"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3676536.3676730"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-024-4155-7"},{"key":"ref10","article-title":"Autochip: Automating hdl generation using llm feedback","author":"Thakur","year":"2023"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/tcad.2024.3483089"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3643681"},{"key":"ref13","article-title":"Ampagent: An llm-based multi-agent system for multi-stage amplifier schematic design from literature for process and performance porting","author":"Liu","year":"2024"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TCAD.2024.3383347"},{"key":"ref15","article-title":"Openroad: Toward a self-driving, open-source digital layout implementation tool chain","volume-title":"Proceedings of Government Microcircuit Applications and Critical Technology Conference","author":"Ajayi"},{"key":"ref16","article-title":"Chipgpt: How far are we from natural language hardware design","author":"Chang","year":"2023"},{"key":"ref17","article-title":"Seed-bench: Benchmarking multimodal llms with generative comprehension","author":"Li","year":"2023"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00913"},{"key":"ref19","first-page":"216","article-title":"Mm-bench: Is your multi-modal model an all-around player?","volume-title":"European Conference on Computer Vision","author":"Liu"},{"key":"ref20","article-title":"Chipexpert: The open-source integrated-circuit-design-specific large language model","author":"Xu","year":"2024"},{"key":"ref21","article-title":"Gpt-4 technical report","author":"Achiam","year":"2023"},{"key":"ref22","article-title":"Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"International conference on machine learning","author":"Li"},{"key":"ref23","article-title":"Qwen-vl: A frontier large vision-language model with versatile abilities","author":"Bai","year":"2023"},{"key":"ref24","article-title":"Internlm-xcomposer: A vision-language large model for advanced text-image comprehension and composition","author":"Zhang","year":"2023"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.02283"},{"key":"ref26","article-title":"Instructblip: Towards general-purpose vision-language models with instruction tuning","author":"Dai","year":"2023"},{"key":"ref27","article-title":"Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"International conference on machine learning","author":"Li"},{"key":"ref28","article-title":"The llama 3 herd of models","author":"Dubey","year":"2024"},{"key":"ref29","article-title":"Minicpm-v: A gpt-4v level mllm on your phone","author":"Yao","year":"2024"},{"key":"ref30","article-title":"Yi: Open foundation models by 01.ai","author":"Young","year":"2024"},{"key":"ref31","article-title":"Kosmos-2: Grounding multimodal large language models to the world","author":"Peng","year":"2023"},{"key":"ref32","article-title":"Gpt-4o system card","year":"2024"},{"key":"ref33","article-title":"Reka core, flash, and edge: A series of powerful multimodal language models","author":"Team","year":"2024"},{"key":"ref34","first-page":"27 730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Advances in neural information processing systems"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.70"},{"key":"ref36","article-title":"Deepseek llm: Scaling open-source language models with longtermism","author":"Bi","year":"2024"},{"key":"ref37","article-title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models","author":"Shao","year":"2024"},{"key":"ref38","article-title":"Deepseek-v2: A strong, economical, and efficient mixture-of-experts language model","author":"Liu","year":"2024"},{"key":"ref39","article-title":"Qwen2 technical report","author":"Yang","year":"2024"},{"key":"ref40","article-title":"Internlm: A multilingual language model with progressively enhanced capabilities","author":"Team","year":"2023"},{"key":"ref41","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref42","article-title":"Minicpm: Unveiling the potential of small language models with scalable training strategies","author":"Hu","year":"2024"},{"key":"ref43","article-title":"Chatglm: A family of large language models from glm-130b to glm-4 all tools","author":"GLM","year":"2024"},{"key":"ref44","article-title":"Gemini: a family of highly capable multimodal models","author":"Team","year":"2023"},{"key":"ref45","article-title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","author":"Team","year":"2024"},{"key":"ref46","article-title":"Introducing the next generation of claude","author":"Team","year":"2024"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1259"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"ref49","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in neural information processing systems"}],"event":{"name":"2025 IEEE\/ACM International Conference On Computer Aided Design (ICCAD)","location":"Munich, Germany","start":{"date-parts":[[2025,10,26]]},"end":{"date-parts":[[2025,10,30]]}},"container-title":["2025 IEEE\/ACM International Conference On Computer Aided Design (ICCAD)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11240608\/11240621\/11240653.pdf?arnumber=11240653","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,21]],"date-time":"2025-11-21T05:42:29Z","timestamp":1763703749000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11240653\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,26]]},"references-count":49,"URL":"https:\/\/doi.org\/10.1109\/iccad66269.2025.11240653","relation":{},"subject":[],"published":{"date-parts":[[2025,10,26]]}}}