{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,24]],"date-time":"2026-09-24T21:37:35Z","timestamp":1790285855893,"version":"4.1.0"},"reference-count":72,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Major Program (JD) of Hubei Province","award":["2023BAA024"],"award-info":[{"award-number":["2023BAA024"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1109\/tse.2025.3595283","type":"journal-article","created":{"date-parts":[[2025,8,19]],"date-time":"2025-08-19T18:15:55Z","timestamp":1755627355000},"page":"3205-3217","source":"Crossref","is-referenced-by-count":15,"title":["Can Large Language Models Serve as Evaluators for Code Summarization?"],"prefix":"10.1109","volume":"51","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-8313-5092","authenticated-orcid":false,"given":"Yang","family":"Wu","sequence":"first","affiliation":[{"name":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6937-4180","authenticated-orcid":false,"given":"Yao","family":"Wan","sequence":"additional","affiliation":[{"name":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4333-8063","authenticated-orcid":false,"given":"Zhaoyang","family":"Chu","sequence":"additional","affiliation":[{"name":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5769-6331","authenticated-orcid":false,"given":"Wenting","family":"Zhao","sequence":"additional","affiliation":[{"name":"Salesforce Research, Palo Alto, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7237-7382","authenticated-orcid":false,"given":"Ye","family":"Liu","sequence":"additional","affiliation":[{"name":"Salesforce Research, Palo Alto, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3063-9425","authenticated-orcid":false,"given":"Hongyu","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Big Data and Software Engineering, Chongqing University, Chongqing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8451-8656","authenticated-orcid":false,"given":"Xuanhua","family":"Shi","sequence":"additional","affiliation":[{"name":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3934-7605","authenticated-orcid":false,"given":"Hai","family":"Jin","sequence":"additional","affiliation":[{"name":"National Engineering Research Center for Big Data Technology and System, Services Computing Technology and System Lab, Cluster and Grid Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3491-5968","authenticated-orcid":false,"given":"Philip S.","family":"Yu","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Illinois Chicago, Chicago, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00087"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3608134"},{"key":"ref4","article-title":"Gpt-4 technical report","year":"2023"},{"key":"ref5","article-title":"Gemini: A family of highly capable multimodal models","author":"Team","year":"2023"},{"key":"ref6","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref8","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Proc. Text Summarization Branches Out","author":"Lin","year":"2004"},{"key":"ref9","first-page":"65","article-title":"METEOR: An automatic metric for MT evaluation with improved correlation with human judgments","volume-title":"Proc. ACL Workshop Intrinsic Extrinsic Eval. Measures Mach. Transl. Summarization","author":"Banerjee","year":"2005"},{"key":"ref10","article-title":"BERTScore: Evaluating text generation with BERT","author":"Zhang","year":"2019"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510060"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468588"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.newsum-1.1"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.870"},{"key":"ref16","doi-asserted-by":"crossref","DOI":"10.18653\/v1\/2023.findings-emnlp.599","article-title":"A closer look into automatic evaluation using large language models","author":"Chiang","year":"2023"},{"key":"ref17","article-title":"ChatEval: Towards better LLM-based evaluators through multi-agent debate","author":"Chan","year":"2023"},{"key":"ref18","article-title":"Large language models are state-of-the-art evaluators of translation quality","author":"Kocmi","year":"2023"},{"key":"ref19","article-title":"Evaluating large language models at evaluating instruction following","author":"Zeng","year":"2023"},{"key":"ref20","first-page":"2232","article-title":"ICE-score: Instructing large language models to evaluate code","volume-title":"Proc. Findings Assoc. Comput. Linguistics (EACL)","author":"Zhuo","year":"2024"},{"key":"ref21","article-title":"ChatGPT"},{"key":"ref22","article-title":"GPT-4o-mini model documentation","year":"2025"},{"key":"ref23","article-title":"GPT-4o model documentation","year":"2025"},{"key":"ref24","article-title":"Deepseek-v3 technical report","author":"DeepSeek","year":"2024"},{"key":"ref25","first-page":"200","article-title":"Deep code comment generation","volume-title":"Proc. 26th Conf. Program Comprehension (ICPC)","author":"Hu","year":"2018"},{"key":"ref26","doi-asserted-by":"crossref","first-page":"1385","DOI":"10.1145\/3377811.3380383","article-title":"Retrieval-based neural source code summarization","volume-title":"Proc. ACM\/IEEE 42nd Int. Conf. Softw. Eng.","author":"Zhang","year":"2020"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.449"},{"key":"ref28","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3650105.3652300"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1006\/csla.1999.0128"},{"key":"ref32","first-page":"349","article-title":"Retrieve and refine: exemplar-based neural comment generation","volume-title":"Proc. 35th IEEE\/ACM Int. Conf. Automated Softw. Eng.","author":"Wei","year":"2020"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC52881.2021.00032"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC52881.2021.00026"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC52881.2021.00049"},{"key":"ref36","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"Liu","year":"2019"},{"key":"ref37","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume-title":"Proc. 36th Int. Conf. Neural Inf. Process. Syst. (NIPS)","author":"Wei","year":"2022"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606763"},{"key":"ref39","article-title":"Multi-agent consensus seeking via large language models","author":"Chen","year":"2023"},{"key":"ref40","article-title":"Gpt-3.5 turbo model documentation","year":"2025"},{"key":"ref41","article-title":"Qwen3"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00373"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.2307\/2332303"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1198\/tech.2001.s620"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/CSMR.2003.1192406"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/585058.585065"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/147001.147033"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2020.2979701"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.37"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238206"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/3664597"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-023-3956-3"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.93"},{"key":"ref54","article-title":"code2seq: Generating sequences from structured representations of code","author":"Alon","year":"2018"},{"key":"ref55","first-page":"746","article-title":"Code to comment \u2018translation\u2019 data, metrics, baselining & evaluation","volume-title":"Proc. 35th IEEE\/ACM Int. Conf. Autom. Softw. Eng.","author":"Gros","year":"2020"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3522674"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678724"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3559548"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639183"},{"key":"ref61","first-page":"56660","article-title":"On-the-fly adapting code summarization on trainable cost-effective language models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Cai","year":"2024"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/s10515-024-00421-4"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/icse55347.2025.00034"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389258"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639174"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.365"},{"key":"ref67","first-page":"6562","article-title":"MLLM-as-a-judge: Assessing multimodal LLM-as-a-judge with vision-language benchmark","volume-title":"Proc. 41st Int. Conf. Mach. Learn","author":"Chen","year":"2024"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-44693-1_54"},{"key":"ref69","first-page":"2638","article-title":"Calibrating LLM-based evaluator","volume-title":"Proc. Joint Int. Conf. Comput. Linguistics, Lang. Resources Eval. (LREC-COLING)","author":"Liu","year":"2024"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.367"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676450"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3516863"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/11298241\/11129179.pdf?arnumber=11129179","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,19]],"date-time":"2025-12-19T08:14:53Z","timestamp":1766132093000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11129179\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":72,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tse.2025.3595283","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,12]]}}}