{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T05:11:25Z","timestamp":1778908285167,"version":"3.51.4"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"8","license":[{"start":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T00:00:00Z","timestamp":1778889600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T00:00:00Z","timestamp":1778889600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1007\/s10489-026-07214-0","type":"journal-article","created":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T04:42:16Z","timestamp":1778906536000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Analyzing the performance of large language models on statement-level code summarization"],"prefix":"10.1007","volume":"56","author":[{"given":"Jie","family":"Zhu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhihui","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tingting","family":"Xu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Junwu","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yonglong","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,5,16]]},"reference":[{"key":"7214_CR1","unstructured":"Zhu Y, Pan M (2019) Automatic code summarization: A systematic literature review. arXiv preprint arXiv:1909.04352"},{"issue":"3","key":"7214_CR2","doi-asserted-by":"publisher","first-page":"471","DOI":"10.3390\/sym14030471","volume":"14","author":"C Zhang","year":"2022","unstructured":"Zhang C, Wang J, Zhou Q, Xu T, Tang K, Gui H, Liu F (2022) A survey of automatic source code summarization. Symmetry 14(3):471","journal-title":"Symmetry"},{"key":"7214_CR3","doi-asserted-by":"crossref","unstructured":"LeClair A, Haque S, Wu L, McMillan C (2020) Improved code summarization via a graph neural network. In: Proceedings of the 28th international conference on program comprehension, pp 184\u2013195","DOI":"10.1145\/3387904.3389268"},{"key":"7214_CR4","unstructured":"Zhang S, Chen Z, Shen Y, Ding M, Tenenbaum JB, Gan C (2023) Planning with large language models for code generation. arXiv preprint arXiv:2303.05510"},{"key":"7214_CR5","doi-asserted-by":"crossref","unstructured":"Khajezade M, Wu JJ, Fard FH, Rodr\u00edguez-P\u00e9rez G, Shehata MS (2024) Investigating the efficacy of large language models for code clone detection. In: Proceedings of the 32nd IEEE\/ACM international conference on program comprehension, pp 161\u2013165","DOI":"10.1145\/3643916.3645030"},{"key":"7214_CR6","doi-asserted-by":"crossref","unstructured":"Sun W, Miao Y, Li Y, Zhang H, Fang C, Liu Y, Deng G, Liu Y, Chen Z (2024) Source code summarization in the era of large language models. arXiv preprint arXiv:2407.07959","DOI":"10.1109\/ICSE55347.2025.00034"},{"issue":"9","key":"7214_CR7","doi-asserted-by":"publisher","first-page":"4268","DOI":"10.1109\/TSE.2023.3279774","volume":"49","author":"A Bansal","year":"2023","unstructured":"Bansal A, Eberhart Z, Karas Z, Huang Y, McMillan C (2023) Function call graph context encoding for neural source code summarization. IEEE Trans Software Eng 49(9):4268\u20134281","journal-title":"IEEE Trans Software Eng"},{"key":"7214_CR8","doi-asserted-by":"crossref","unstructured":"Shi L, Mu F, Chen X, Wang S, Wang J, Yang Y, Li G, Xia X, Wang Q (2022) Are we building on the rock? on the importance of data preprocessing for code summarization. In: Proceedings of the 30th ACM joint European software engineering conference and symposium on the foundations of software engineering, pp 107\u2013119","DOI":"10.1145\/3540250.3549145"},{"key":"7214_CR9","unstructured":"Dong Q, Li L, Dai D, Zheng C, Wu Z, Chang B, Sun X, Xu J, Sui Z (2022) A survey on in-context learning. arXiv preprint arXiv:2301.00234"},{"key":"7214_CR10","doi-asserted-by":"crossref","unstructured":"Papineni K, Roukos S, Ward T, Zhu W-J (2002) Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting of the association for computational linguistics, pp 311\u2013318","DOI":"10.3115\/1073083.1073135"},{"key":"7214_CR11","unstructured":"Banerjee S, Lavie A (2005) Meteor: An automatic metric for mt evaluation with improved correlation with human judgments. In: Proceedings of the Acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization, pp 65\u201372"},{"key":"7214_CR12","unstructured":"Lin C-Y (2004) Rouge: A package for automatic evaluation of summaries. In: Text summarization branches out, pp 74\u201381"},{"key":"7214_CR13","doi-asserted-by":"crossref","unstructured":"Reimers N, Gurevych I (2019) Sentence-bert: Sentence embeddings using siamese bert-networks. arXiv preprint arXiv:1908.10084","DOI":"10.18653\/v1\/D19-1410"},{"key":"7214_CR14","unstructured":"White J, Fu Q, Hays S, Sandborn M, Olea C, Gilbert H, Elnashar A, Spencer-Smith J, Schmidt DC (2023) A prompt pattern catalog to enhance prompt engineering with chatgpt. arXiv preprint arXiv:2302.11382"},{"issue":"2","key":"7214_CR15","first-page":"1","volume":"10","author":"W Wang","year":"2019","unstructured":"Wang W, Zheng VW, Yu H, Miao C (2019) A survey of zero-shot learning: Settings, methods, and applications. ACM Trans Intell Syst Technol (TIST) 10(2):1\u201337","journal-title":"ACM Trans Intell Syst Technol (TIST)"},{"key":"7214_CR16","doi-asserted-by":"crossref","unstructured":"Yao B, Chen G, Zou R, Lu Y, Li J, Zhang S, Liu S, Hendler J, Wang D (2023) More samples or more prompt inputs? exploring effective in-context sampling for llm few-shot prompt engineering. arXiv preprint arXiv:2311.09782","DOI":"10.18653\/v1\/2024.findings-naacl.115"},{"key":"7214_CR17","doi-asserted-by":"crossref","unstructured":"Kim G, Baldi P, McAleer S (2024) Language models can solve computer tasks. Adv Neural Inf Process Syst 36","DOI":"10.52202\/075280-1723"},{"key":"7214_CR18","first-page":"24824","volume":"35","author":"J Wei","year":"2022","unstructured":"Wei J, Wang X, Schuurmans D, Bosma M, Xia F, Chi E, Le QV, Zhou D et al (2022) Chain-of-thought prompting elicits reasoning in large language models. Adv Neural Inf Process Syst 35:24824\u201324837","journal-title":"Adv Neural Inf Process Syst"},{"key":"7214_CR19","doi-asserted-by":"crossref","unstructured":"Islam R, Moushi OM (2024) Gpt-4o: The cutting-edge advancement in multimodal llm. Authorea Preprints","DOI":"10.36227\/techrxiv.171986596.65533294\/v1"},{"key":"7214_CR20","unstructured":"Touvron H, Lavril T, Izacard G, Martinet X, Lachaux M-A, Lacroix T, Rozi\u00e8re B, Goyal N, Hambro E, Azhar F et al (2023) Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971"},{"key":"7214_CR21","unstructured":"Team G, Mesnard T, Hardin C, Dadashi R, Bhupatiraju S, Pathak S, Sifre L, Rivi\u00e8re M, Kale MS, Love J et al (2024) Gemma: Open models based on gemini research and technology. arXiv preprint arXiv:2403.08295"},{"key":"7214_CR22","doi-asserted-by":"crossref","unstructured":"Huang H, Qu Y, Liu J, Yang M, Zhao T (2024) An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task-specific classifiers. arXiv preprint arXiv:2403.02839","DOI":"10.18653\/v1\/2025.findings-acl.306"},{"issue":"4","key":"7214_CR23","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen J (1968) Weighted kappa: Nominal scale agreement provision for scaled disagreement or partial credit. Psychol Bull 70(4):213","journal-title":"Psychol Bull"},{"key":"7214_CR24","unstructured":"Zhu J LLM4Sta-Codesummarization. https:\/\/github.com\/smallburningpig\/LLM4Sta-Codesummarization"},{"key":"7214_CR25","doi-asserted-by":"crossref","unstructured":"Steidl D, Hummel B, Juergens E (2013) Quality analysis of source code comments. In: ICPC, pp 83\u201392. IEEE","DOI":"10.1109\/ICPC.2013.6613836"},{"key":"7214_CR26","doi-asserted-by":"publisher","first-page":"545","DOI":"10.1109\/TSE.1980.234503","volume":"6","author":"SS Yau","year":"1980","unstructured":"Yau SS, Collofello JS (1980) Some stability measures for software maintenance. IEEE Trans Software Eng 6:545\u2013552","journal-title":"IEEE Trans Software Eng"},{"key":"7214_CR27","doi-asserted-by":"crossref","unstructured":"Wang Y, Wang W, Joty S, Hoi SC (2021) Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"7214_CR28","doi-asserted-by":"crossref","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D et al (2020) Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"issue":"1","key":"7214_CR29","doi-asserted-by":"publisher","first-page":"104342","DOI":"10.1016\/j.ipm.2025.104342","volume":"63","author":"PN Ahmad","year":"2026","unstructured":"Ahmad PN, Shah AM, Lee K, Muhammad W (2026) Misinformation detection on online social networks using pretrained language models. Inf Process Manag 63(1):104342. https:\/\/doi.org\/10.1016\/j.ipm.2025.104342","journal-title":"Inf Process Manag"},{"key":"7214_CR30","unstructured":"Sun W, Fang C, You Y, Miao Y, Liu Y, Li Y, Deng G, Huang S, Chen Y, Zhang Q, Qian H, Liu Y, Chen Z (2023) Automatic code summarization via chatgpt: How far are we?, 1\u201313 CoRR arxiv:2305.12865"},{"key":"7214_CR31","doi-asserted-by":"crossref","unstructured":"Ahmed T, Devanbu P (2022) Few-shot training llms for project-specific code-summarization. In: Proceedings of the 37th IEEE\/ACM international conference on automated software engineering, pp 1\u20135","DOI":"10.1145\/3551349.3559555"},{"key":"7214_CR32","unstructured":"Fried D, Aghajanyan A, Lin J, Wang S, Wallace E, Shi F, Zhong R, Yih W-t, Zettlemoyer L, Lewis M (2022) Incoder: A generative model for code infilling and synthesis. arXiv preprint arXiv:2204.05999"},{"key":"7214_CR33","unstructured":"Husain H, Wu H-H, Gazit T, Allamanis M, Brockschmidt M (2019) Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436"},{"key":"7214_CR34","doi-asserted-by":"crossref","unstructured":"Zhu J, Miao Y, Xu T, Zhu J, Sun X (2024) On the effectiveness of large language models in statement-level code summarization. In: 2024 IEEE 24th International Conference on Software Quality, Reliability and Security (QRS), pp 216\u2013227. IEEE","DOI":"10.1109\/QRS62785.2024.00030"},{"key":"7214_CR35","doi-asserted-by":"crossref","unstructured":"Chandra L, Susan S, Kumar D, Kant K (2024) Unveiling toxic tendencies of small language models in unconstrained generation tasks. In: 2024 IEEE International Conference on Electronics, Computing and Communication Technologies (CONECCT), pp 1\u20136. IEEE","DOI":"10.1109\/CONECCT62155.2024.10677188"},{"key":"7214_CR36","doi-asserted-by":"crossref","unstructured":"Devlin J, Chang M, Lee K, Toutanova K (2019) BERT: pre-training of deep bidirectional transformers for language understanding. In: NAACL-HLT, pp 4171\u20134186. ACL, Minneapolis, MN, USA","DOI":"10.18653\/v1\/N19-1423"},{"key":"7214_CR37","doi-asserted-by":"crossref","unstructured":"Gao S, Wen X-C, Gao C, Wang W, Zhang H, Lyu MR (2023) What makes good in-context demonstrations for code intelligence tasks with llms? In: 2023 38th IEEE\/ACM International Conference on Automated Software Engineering (ASE), pp 761\u2013773. IEEE","DOI":"10.1109\/ASE56229.2023.00109"},{"key":"7214_CR38","doi-asserted-by":"crossref","unstructured":"Shi E, Wang Y, Du L, Chen J, Han S, Zhang H, Zhang D, Sun H (2022) On the evaluation of neural code summarization. In: Proceedings of the 44th international conference on software engineering, pp 1597\u20131608. IEEE, Pittsburgh, USA","DOI":"10.1145\/3510003.3510060"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-026-07214-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-026-07214-0","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-026-07214-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T04:43:05Z","timestamp":1778906585000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-026-07214-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,16]]},"references-count":38,"journal-issue":{"issue":"8","published-print":{"date-parts":[[2026,6]]}},"alternative-id":["7214"],"URL":"https:\/\/doi.org\/10.1007\/s10489-026-07214-0","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5,16]]},"assertion":[{"value":"12 December 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 March 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 May 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"259"}}