{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T04:08:24Z","timestamp":1754021304989,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031705625"},{"type":"electronic","value":"9783031705632"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-70563-2_17","type":"book-chapter","created":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T22:29:51Z","timestamp":1725143391000},"page":"214-226","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Evaluation Metrics in\u00a0LLM Code Generation"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-6290-9757","authenticated-orcid":false,"given":"Kai","family":"Hartung","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sambit","family":"Mallick","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"S\u00f6ren","family":"Gr\u00f6ttrup","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Munir","family":"Georges","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,1]]},"reference":[{"key":"17_CR1","unstructured":"Code metrics values. https:\/\/learn.microsoft.com\/en-us\/visualstudio\/code-quality\/code-metrics-values?view=vs-2022. Accessed 13 Feb 2023"},{"key":"17_CR2","doi-asserted-by":"publisher","unstructured":"Abu-Aisheh, Z., Raveaux, R., Ramel, J.Y., Martineau, P.: An exact graph edit distance algorithm for solving pattern recognition problems. In: 4th International Conference on Pattern Recognition Applications and Methods 2015, Lisbon, Portugal (2015). https:\/\/doi.org\/10.5220\/0005209202710278. https:\/\/hal.science\/hal-01168816","DOI":"10.5220\/0005209202710278"},{"key":"17_CR3","unstructured":"Austin, J., et\u00a0al.: Program synthesis with large language models. arXiv preprint arXiv:2108.07732 (2021)"},{"key":"17_CR4","doi-asserted-by":"crossref","unstructured":"Bafatakis, N., et al.: Python coding style compliance on stack overflow. In: 2019 IEEE\/ACM 16th International Conference on Mining Software Repositories (MSR), pp. 210\u2013214. IEEE (2019)","DOI":"10.1109\/MSR.2019.00042"},{"issue":"4","key":"17_CR5","doi-asserted-by":"publisher","first-page":"546","DOI":"10.1109\/TSE.2009.70","volume":"36","author":"RP Buse","year":"2009","unstructured":"Buse, R.P., Weimer, W.R.: Learning a metric for code readability. IEEE Trans. Softw. Eng. 36(4), 546\u2013558 (2009)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"17_CR6","unstructured":"Chen, M., et\u00a0al.: Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374 (2021)"},{"issue":"6","key":"17_CR7","doi-asserted-by":"publisher","first-page":"158","DOI":"10.1007\/s10664-022-10193-8","volume":"27","author":"S Chowdhury","year":"2022","unstructured":"Chowdhury, S., Holmes, R., Zaidman, A., Kazman, R.: Revisiting the debate: are code metrics useful for measuring maintenance effort? Empir. Softw. Eng. 27(6), 158 (2022)","journal-title":"Empir. Softw. Eng."},{"issue":"8","key":"17_CR8","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1109\/2.303623","volume":"27","author":"D Coleman","year":"1994","unstructured":"Coleman, D., Ash, D., Lowther, B., Oman, P.: Using metrics to evaluate software system maintainability. Computer 27(8), 44\u201349 (1994)","journal-title":"Computer"},{"key":"17_CR9","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111741","volume":"203","author":"M Evtikhiev","year":"2023","unstructured":"Evtikhiev, M., Bogomolov, E., Sokolov, Y., Bryksin, T.: Out of the BLEU: how should we assess quality of the Code Generation models? J. Syst. Softw. 203, 111741 (2023)","journal-title":"J. Syst. Softw."},{"key":"17_CR10","doi-asserted-by":"crossref","unstructured":"Feng, Z., et\u00a0al.: Codebert: a pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155 (2020)","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"17_CR11","unstructured":"Guido van Rossum, Barry Warsaw, Alyssa Coghlan: PEP 8 - Style Guide for Python Code (2001). https:\/\/peps.python.org\/pep-0008\/"},{"key":"17_CR12","unstructured":"Gunasekar, S., et al.: Textbooks Are All You Need (2023). https:\/\/arxiv.org\/pdf\/2306.11644v2.pdf. _eprint: 2306.11644"},{"key":"17_CR13","doi-asserted-by":"crossref","unstructured":"Hagberg, A.A., Schult, D.A., Swart, P.J.: Exploring network structure, dynamics, and function using networkX. In: Varoquaux, G., Vaught, T., Millman, J. (eds.) Proceedings of the 7th Python in Science Conference, Pasadena, CA, USA, pp. 11 \u2013 15 (2008)","DOI":"10.25080\/TCWV9851"},{"key":"17_CR14","unstructured":"Halstead, M.H.: Elements of Software Science (Operating and programming systems series). Elsevier Science Inc. (1977)"},{"key":"17_CR15","doi-asserted-by":"publisher","unstructured":"Heitlager, I., Kuipers, T., Visser, J.: A practical model for measuring maintainability. In: 6th International Conference on the Quality of Information and Communications Technology (QUATIC 2007), pp. 30\u201339 (2007). https:\/\/doi.org\/10.1109\/QUATIC.2007.8. https:\/\/ieeexplore.ieee.org\/stamp\/stamp.jsp?tp=&arnumber=4335232","DOI":"10.1109\/QUATIC.2007.8"},{"key":"17_CR16","unstructured":"Li, Y., Bubeck, S., Eldan, R., Giorno, A.D., Gunasekar, S., Lee, Y.T.: Textbooks are all you need ii: phi-1.5 technical report (2023)"},{"issue":"6624","key":"17_CR17","doi-asserted-by":"publisher","first-page":"1092","DOI":"10.1126\/science.abq1158","volume":"378","author":"Y Li","year":"2022","unstructured":"Li, Y., et al.: Competition-level code generation with alphacode. Science 378(6624), 1092\u20131097 (2022)","journal-title":"Science"},{"key":"17_CR18","unstructured":"Liu, J., Xia, C.S., Wang, Y., ZHANG, L.: Is your code generated by ChatGPT really correct? rigorous evaluation of large language models for code generation. In: Oh, A., Neumann, T., Globerson, A., Saenko, K., Hardt, M., Levine, S. (eds.) Advances in Neural Information Processing Systems, vol.\u00a036, pp. 21558\u201321572. Curran Associates, Inc. (2023)"},{"key":"17_CR19","doi-asserted-by":"publisher","unstructured":"McCabe, T.: A complexity measure. IEEE Trans. Softw. Eng. SE-2(4), 308\u2013320 (1976). https:\/\/doi.org\/10.1109\/TSE.1976.233837. https:\/\/ieeexplore.ieee.org\/stamp\/stamp.jsp?tp=&arnumber=1702388","DOI":"10.1109\/TSE.1976.233837"},{"key":"17_CR20","unstructured":"Lacchia, M.: Introduction to Code Metrics (2020). https:\/\/radon.readthedocs.io\/en\/latest\/intro.html"},{"key":"17_CR21","unstructured":"Nijkamp, E., et al.: Codegen: an open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474 (2022)"},{"key":"17_CR22","unstructured":"Achiam, J., et\u00a0al.: OpenAI: Gpt-4 technical report (2023)"},{"key":"17_CR23","doi-asserted-by":"publisher","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: Bleu: a method for automatic evaluation of machine translation. In: Isabelle, P., Charniak, E., Lin, D. (eds.) Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics, pp. 311\u2013318. Association for Computational Linguistics, Philadelphia (2002). https:\/\/doi.org\/10.3115\/1073083.1073135. https:\/\/aclanthology.org\/P02-1040","DOI":"10.3115\/1073083.1073135"},{"key":"17_CR24","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I., et\u00a0al.: Improving language understanding by generative pre-training (2018)"},{"key":"17_CR25","doi-asserted-by":"crossref","unstructured":"Reimers, N., Gurevych, I.: Sentence-bert: sentence embeddings using siamese bert-networks (2019). https:\/\/arxiv.org\/abs\/1908.10084","DOI":"10.18653\/v1\/D19-1410"},{"key":"17_CR26","unstructured":"Ren, S., et al.: CodeBLEU: a method for automatic evaluation of code synthesis (2020). https:\/\/arxiv.org\/pdf\/2009.10297.pdf. _eprint: 2009.10297"},{"key":"17_CR27","unstructured":"Rozi\u00e8re, B., et al.: Code Llama: open foundation models for code (2023). https:\/\/arxiv.org\/pdf\/2308.12950.pdf. _eprint: 2308.12950"},{"key":"17_CR28","doi-asserted-by":"crossref","unstructured":"Wang, Y., Wang, W., Joty, S., Hoi, S.C.: Codet5: identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv preprint arXiv:2109.00859 (2021)","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"17_CR29","unstructured":"Wei, Y., Wang, Z., Liu, J., Ding, Y., Zhang, L.: Magicoder: source code is all you need (2023)"}],"container-title":["Lecture Notes in Computer Science","Text, Speech, and Dialogue"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-70563-2_17","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T22:35:02Z","timestamp":1725143702000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-70563-2_17"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031705625","9783031705632"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-70563-2_17","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"1 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"TSD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Text, Speech, and Dialogue","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Brno","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Czech Republic","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"tsd2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.tsdconference.org\/tsd2024\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}