{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T06:57:13Z","timestamp":1781161033875,"version":"3.54.1"},"publisher-location":"Singapore","reference-count":30,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819200672","type":"print"},{"value":"9789819200689","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-981-92-0068-9_12","type":"book-chapter","created":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T06:08:21Z","timestamp":1781158101000},"page":"166-183","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["How Smart is Smart Enough? Benchmarking LLMs with\u00a0Embedding-Based Similarity in\u00a0Python Code Generation"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-4883-0516","authenticated-orcid":false,"given":"Dominik","family":"Palla","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8000-9766","authenticated-orcid":false,"given":"Blanka","family":"Klimova","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5633-9332","authenticated-orcid":false,"given":"Marcel","family":"Pikhart","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9011-6987","authenticated-orcid":false,"given":"Eva","family":"Svejdarova","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,1]]},"reference":[{"key":"12_CR1","doi-asserted-by":"publisher","unstructured":"Aarti: Generative AI in software development: an overview and evaluation of modern coding tools. Int. J. Multidisciplinary Res. (IJFMR) 6(3) (2024). https:\/\/doi.org\/10.36948\/ijfmr.2024.v06i03.23271","DOI":"10.36948\/ijfmr.2024.v06i03.23271"},{"key":"12_CR2","doi-asserted-by":"publisher","unstructured":"Bandi, A., Adapa, P.V.S.R., Kuchi, Y.E.V.P.K.: The power of generative AI: a review of requirements, models, input\u2013output formats, evaluation metrics, and challenges. Future Internet 15(8), 260 (2023). https:\/\/doi.org\/10.3390\/fi15080260","DOI":"10.3390\/fi15080260"},{"issue":"6","key":"12_CR3","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1109\/MS.2024.3418570","volume":"41","author":"B Baudry","year":"2024","unstructured":"Baudry, B., et al.: Generative AI to generate test data generators. IEEE Softw. 41(6), 55\u201364 (2024). https:\/\/doi.org\/10.1109\/MS.2024.3418570","journal-title":"IEEE Softw."},{"key":"12_CR4","doi-asserted-by":"publisher","unstructured":"Chiarello, F., Giordano, V., Spada, I., Barandoni, S., Fantoni, G.: Future applications of generative large language models: a data-driven case study on chatgpt. Technovation 133, 103002 (2024). https:\/\/doi.org\/10.1016\/j.technovation.2024.103002","DOI":"10.1016\/j.technovation.2024.103002"},{"key":"12_CR5","doi-asserted-by":"publisher","unstructured":"Dakhel, A.M., Nikanjam, A., Khomh, F., Desmarais, M.C., Washizaki, H.: Generative AI for software development: a family of studies on code generation. In: Nguyen-Duc, A., Abrahamsson, P., Khomh, F. (eds.) Generative AI for Effective Software Development, pp. 151\u2013172. Springer, Cham (2024). https:\/\/doi.org\/10.1007\/978-3-031-55642-5_7","DOI":"10.1007\/978-3-031-55642-5_7"},{"key":"12_CR6","unstructured":"Evtikhiev, S., Ivanov, A., Svyatkovskiy, A.: How well do automatic metrics correlate with human evaluation in code generation? J. Syst. Softw. (2023)"},{"issue":"5","key":"12_CR7","doi-asserted-by":"publisher","first-page":"649","DOI":"10.1016\/j.bushor.2024.05.009","volume":"67","author":"SL France","year":"2024","unstructured":"France, S.L.: Navigating software development in the chatgpt and github copilot era. Bus. Horiz. 67(5), 649\u2013661 (2024). https:\/\/doi.org\/10.1016\/j.bushor.2024.05.009","journal-title":"Bus. Horiz."},{"key":"12_CR8","doi-asserted-by":"publisher","unstructured":"Grilo, D., et al.: Chatgpt\u2019s potential in answering radiotherapy treatment-related questions: an observational study comparing responses to human experts. Cancer 1(e63677) (2025). https:\/\/doi.org\/10.1002\/cncr.63677","DOI":"10.1002\/cncr.63677"},{"issue":"2","key":"12_CR9","doi-asserted-by":"publisher","first-page":"62","DOI":"10.3390\/a17020062","volume":"17","author":"B Idrisov","year":"2024","unstructured":"Idrisov, B., Schlippe, T.: Program code generation with generative AIs. Algorithms 17(2), 62 (2024). https:\/\/doi.org\/10.3390\/a17020062","journal-title":"Algorithms"},{"key":"12_CR10","unstructured":"Liu, J., Wang, Y.: Evalplus: automated test augmentation for better code evaluation. arXiv preprint arXiv:2306.00979 (2023)"},{"key":"12_CR11","doi-asserted-by":"publisher","unstructured":"Mahmud, T., Duan, B., Pasareanu, C., Yang, G.: Enhancing LLM code generation with ensembles: a similarity-based selection approach. arXiv preprint (2025). https:\/\/doi.org\/10.48550\/arXiv.2503.15838","DOI":"10.48550\/arXiv.2503.15838"},{"key":"12_CR12","doi-asserted-by":"publisher","unstructured":"McIntosh, T.R., et al.: From google Gemini to OpenAI q* (q-star): a survey on reshaping the generative artificial intelligence (AI) research landscape. Technologies 13(2), 51 (2025). https:\/\/doi.org\/10.3390\/technologies13020051. https:\/\/doi.org\/10.3390\/technologies13020051","DOI":"10.3390\/technologies13020051"},{"key":"12_CR13","unstructured":"OpenAI: ChatGPT (GPT-4o) (2025). https:\/\/chat.openai.com. Accessed 15 May 2025"},{"key":"12_CR14","unstructured":"Patel, N., Rajan, A.: AIME: AI model evaluation through ensemble LLMs. IEEE Trans. Softw. Eng. (2024)"},{"issue":"12","key":"12_CR15","doi-asserted-by":"publisher","first-page":"123","DOI":"10.9734\/ajrcos\/2024\/v17i12533","volume":"17","author":"S Ramalakshmi","year":"2024","unstructured":"Ramalakshmi, S., Asha, G.: Exploring generative AI: models, applications, and challenges in data synthesis. Asian J. Res. Comput. Sci. 17(12), 123\u2013136 (2024). https:\/\/doi.org\/10.9734\/ajrcos\/2024\/v17i12533","journal-title":"Asian J. Res. Comput. Sci."},{"key":"12_CR16","doi-asserted-by":"publisher","unstructured":"Rani, G., Singh, J., Khanna, A.: Comparative analysis of generative AI models. In: 2023 International Conference on Advances in Computation, Communication and Information Technology (ICAICCIT), pp. 760\u2013765 (2023). https:\/\/doi.org\/10.1109\/ICAICCIT60255.2023.10465941","DOI":"10.1109\/ICAICCIT60255.2023.10465941"},{"key":"12_CR17","unstructured":"Rollman, J.C., et\u00a0al.: Practical design and benchmarking of generative AI applications for surgical billing and coding. arXiv preprint (2025). https:\/\/arxiv.org\/abs\/2501.05479"},{"key":"12_CR18","doi-asserted-by":"publisher","unstructured":"Sajja, A., Thakur, D., Mehra, A.: Integrating generative AI into the software development lifecycle: impacts on code quality and maintenance. Int. J. Sci. Res. Arch. 13(1), 1952\u20131960 (2024). https:\/\/doi.org\/10.30574\/ijsra.2024.13.1.1837","DOI":"10.30574\/ijsra.2024.13.1.1837"},{"issue":"1","key":"12_CR19","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1007\/s10515-024-00426-z","volume":"31","author":"J Sauvola","year":"2024","unstructured":"Sauvola, J., Tarkoma, S., Klemettinen, M., Riekki, J., Doermann, D.: Future of software development with generative AI. Autom. Softw. Eng. 31(1), 26 (2024). https:\/\/doi.org\/10.1007\/s10515-024-00426-z","journal-title":"Autom. Softw. Eng."},{"issue":"1","key":"12_CR20","doi-asserted-by":"publisher","first-page":"2439610","DOI":"10.1080\/08839514.2024.2439610","volume":"39","author":"A Sobo","year":"2025","unstructured":"Sobo, A., Awes, M., Almas, B., Polatidis, N.: Evaluating LLMs for code generation in HRI: a comparative study of chatgpt, gemini, and claude. Appl. Artif. Intell. 39(1), 2439610 (2025). https:\/\/doi.org\/10.1080\/08839514.2024.2439610","journal-title":"Appl. Artif. Intell."},{"key":"12_CR21","unstructured":"Tong, W., Zhang, L.: Codejudge: a self-evaluating framework for AI-generated code. In: EMNLP (2024)"},{"issue":"6","key":"12_CR22","doi-asserted-by":"publisher","first-page":"188","DOI":"10.3390\/fi16060188","volume":"16","author":"D Tosi","year":"2024","unstructured":"Tosi, D.: Studying the quality of source code generated by different AI generative engines: An empirical evaluation. Future Internet 16(6), 188 (2024). https:\/\/doi.org\/10.3390\/fi16060188","journal-title":"Future Internet"},{"key":"12_CR23","doi-asserted-by":"publisher","unstructured":"Wang, B., Kuo, C.C.J., Li, H.: Just rank: rethinking evaluation with word and sentence similarities. In: Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (ACL), pp. 6060\u20136077. Dublin, Ireland (2022). https:\/\/doi.org\/10.18653\/v1\/2022.acl-long.419","DOI":"10.18653\/v1\/2022.acl-long.419"},{"key":"12_CR24","doi-asserted-by":"publisher","unstructured":"Wang, R., Guo, J., Gao, C., Fan, G., Chong, C.Y., Xia, X.: Can LLMs replace human evaluators? An empirical study of LLM-as-a-judge in software engineering. arXiv preprint (2025). https:\/\/doi.org\/10.48550\/arXiv.2502.06193","DOI":"10.48550\/arXiv.2502.06193"},{"key":"12_CR25","unstructured":"Wu, Z., et\u00a0al.: Bridge: benchmarking large language models for understanding clinical practice guidelines. arXiv preprint arXiv:2404.02939 (2024)"},{"key":"12_CR26","doi-asserted-by":"publisher","unstructured":"Yadav, A., Beniwal, H., Singh, M.: Pythonsaga: redefining the benchmark to evaluate code generating LLMs. In: Findings of the Association for Computational Linguistics: EMNLP 2024, pp. 17113\u201317126. Miami, FL (2024). https:\/\/doi.org\/10.18653\/v1\/2024.findings-emnlp.996","DOI":"10.18653\/v1\/2024.findings-emnlp.996"},{"key":"12_CR27","doi-asserted-by":"publisher","unstructured":"Yadav, A., Beniwal, H., Singh, M.: PythonSaga: redefining the benchmark to evaluate code generating LLMs. In: Al-Onaizan, Y., Bansal, M., Chen, Y.N. (eds.) Findings of the Association for Computational Linguistics: EMNLP 2024, pp. 17113\u201317126. Association for Computational Linguistics, Miami, Florida, USA (2024). https:\/\/doi.org\/10.18653\/v1\/2024.findings-emnlp.996. https:\/\/aclanthology.org\/2024.findings-emnlp.996\/","DOI":"10.18653\/v1\/2024.findings-emnlp.996"},{"key":"12_CR28","doi-asserted-by":"publisher","unstructured":"Zhai, C., Wibowo, S., Li, L.D.: The effects of over-reliance on ai dialogue systems on students\u2019 cognitive abilities: a systematic review. Smart Learn. Environ. 11, 28 (2024). https:\/\/doi.org\/10.1186\/s40561-024-00316-7","DOI":"10.1186\/s40561-024-00316-7"},{"key":"12_CR29","unstructured":"Zheng, S., Tan, H.: LLM-as-a-judge: Language models for evaluation. In: NeurIPS (2024)"},{"key":"12_CR30","doi-asserted-by":"crossref","unstructured":"Zhou, Z., Chen, X., Baral, C.: Codebertscore: evaluating code generation with pretrained embeddings. arXiv preprint arXiv:2301.13136 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.859"}],"container-title":["Communications in Computer and Information Science","Recent Challenges in Intelligent information and Database Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-92-0068-9_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T06:08:28Z","timestamp":1781158108000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-92-0068-9_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9789819200672","9789819200689"],"references-count":30,"URL":"https:\/\/doi.org\/10.1007\/978-981-92-0068-9_12","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"value":"1865-0929","type":"print"},{"value":"1865-0937","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"1 June 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ACIIDS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Asian Conference on Intelligent Information and Database Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Kaohsiung","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Taiwan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 April 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aciids2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aciids.pwr.edu.pl\/2026\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}