{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,9]],"date-time":"2025-12-09T11:21:52Z","timestamp":1765279312812,"version":"3.41.0"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031945779","type":"print"},{"value":"9783031945786","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-94578-6_16","type":"book-chapter","created":{"date-parts":[[2025,5,30]],"date-time":"2025-05-30T05:14:37Z","timestamp":1748582077000},"page":"280-296","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["LLM-KG-Bench 3.0: A Compass for\u00a0Semantic Technology Capabilities in\u00a0the\u00a0Ocean of\u00a0LLMs"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5260-5181","authenticated-orcid":false,"given":"Lars-Peter","family":"Meyer","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3127-0815","authenticated-orcid":false,"given":"Johannes","family":"Frey","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4486-3046","authenticated-orcid":false,"given":"Desiree","family":"Heim","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-5245-6655","authenticated-orcid":false,"given":"Felix","family":"Brei","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9948-6458","authenticated-orcid":false,"given":"Claus","family":"Stadler","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1337-2770","authenticated-orcid":false,"given":"Kurt","family":"Junghanns","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0762-8688","authenticated-orcid":false,"given":"Michael","family":"Martin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,5,31]]},"reference":[{"key":"16_CR1","unstructured":"Abdin, M., et\u00a0al.: Phi-3 technical report: a highly capable language model locally on your phone. arXiv:2404.14219 (2024)"},{"key":"16_CR2","doi-asserted-by":"publisher","unstructured":"Bai, G., et al.: MT-bench-101: a fine-grained benchmark for evaluating large language models in multi-turn dialogues. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 7421\u20137454. Association for Computational Linguistics (2024). https:\/\/doi.org\/10.18653\/v1\/2024.acl-long.401","DOI":"10.18653\/v1\/2024.acl-long.401"},{"key":"16_CR3","unstructured":"Brei, F., Frey, J., Meyer, L.P.: Leveraging small language models for text2sparqltasks to improve the resilience of AI assistance. In: Holze, J., Tramp, S., Martin, M., Auer, S., Usbeck, R., Krdzavac, N. (eds.) Proceedings of the Third International Workshop on Linked Data-driven Resilience Research 2024 (D2R2 2024), colocated with ESWC 2024. CEUR Workshop Proceedings, vol.\u00a03707. CEUR-WS.org (2024). https:\/\/ceur-ws.org\/Vol-3707\/D2R224_paper_5.pdf"},{"key":"16_CR4","doi-asserted-by":"publisher","unstructured":"Brei, F., Meyer, L.P., Martin, M.: Queryfy: from knowledge graphs to questions using open large language models: enabling finetuning by question generation on given knowledge. it - Information Technology (2025). https:\/\/doi.org\/10.1515\/itit-2024-0079","DOI":"10.1515\/itit-2024-0079"},{"key":"16_CR5","unstructured":"Chiang, W.L., et al.: Chatbot arena: an open platform for evaluating LLMs by human preference. In: Salakhutdinov, R., et al. (eds.) Proceedings of the 41st International Conference on Machine Learning. Proceedings of Machine Learning Research, vol.\u00a0235, pp. 8359\u20138388. PMLR (2024)"},{"key":"16_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1007\/978-3-030-30796-7_5","volume-title":"The Semantic Web \u2013 ISWC 2019","author":"M Dubey","year":"2019","unstructured":"Dubey, M., Banerjee, D., Abdelkawi, A., Lehmann, J.: LC-QuAD 2.0: a large dataset for complex question answering over Wikidata and DBpedia. In: Ghidini, C., et al. (eds.) ISWC 2019. LNCS, vol. 11779, pp. 69\u201378. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-30796-7_5"},{"key":"16_CR7","unstructured":"Fourrier, C., Habib, N., Lozovskaya, A., Szafer, K., Wolf, T.: Open LLM leaderboard v2 (2024). https:\/\/huggingface.co\/spaces\/open-llm-leaderboard\/open_llm_leaderboard"},{"key":"16_CR8","unstructured":"Frey, J., Meyer, L., Arndt, N., Brei, F., Bulert, K.: Benchmarking the abilities of large language models for RDF knowledge graph creation and comprehension: How well do LLMs speak turtle? In: Alam, M., Cochez, M. (eds.) Proceedings of the Workshop on Deep Learning for Knowledge Graphs (DL4KG 2023) co-located with the 21th International Semantic Web Conference (ISWC 2023), Athens, 6\u201310 November 2023. CEUR Workshop Proceedings, vol.\u00a03559. CEUR-WS.org (2023). https:\/\/ceur-ws.org\/Vol-3559\/paper-3.pdf"},{"key":"16_CR9","doi-asserted-by":"publisher","unstructured":"Frey, J., Meyer, L.P., Brei, F., Gruender, S., Martin, M.: Assessing the evolution of LLM capabilities for knowledge graph engineering in 2023. In: The Semantic Web: ESWC 2024 Satellite Events, pp. 51\u201360. Springer, Cham (2025). https:\/\/doi.org\/10.1007\/978-3-031-78952-6_5","DOI":"10.1007\/978-3-031-78952-6_5"},{"key":"16_CR10","unstructured":"Grattafiori, A., et\u00a0al.: The llama 3 herd of models. arXiv:2407.21783 (2024)"},{"key":"16_CR11","unstructured":"Guo, D., et al.: Deepseek-coder: when the large language model meets programming \u2013 the rise of code intelligence. arXiv:2401.14196 (2024)"},{"key":"16_CR12","unstructured":"Hofer, M., Frey, J., Rahm, E.: Towards self-configuring knowledge graph construction pipelines using LLMs - a case study with RML. In: Fifth International Workshop on Knowledge Graph Construction @ ESWC2024. CEUR Workshop Proceedings, vol.\u00a03718. CEUR-WS.org (2024). https:\/\/ceur-ws.org\/Vol-3718\/paper6.pdf"},{"key":"16_CR13","unstructured":"Huang, S., et al.: Opencoder: the open cookbook for top-tier code large language models. arXiv:2411.04905 (2024)"},{"key":"16_CR14","unstructured":"Hui, B., et\u00a0al.: Qwen2.5-coder technical report. arXiv:2409.12186 (2024)"},{"key":"16_CR15","unstructured":"Kovriguina, L., Teucher, R., Radyush, D., Mouromtsev, D.: Sparqlgen: one-shot prompt-based approach for sparql query generation. In: International Conference on Semantic Systems. CEUR Workshop Proceedings, vol.\u00a03526. CEUR-WS.org (2023). https:\/\/ceur-ws.org\/Vol-3526\/paper-08.pdf"},{"key":"16_CR16","doi-asserted-by":"crossref","unstructured":"Kwon, W., et al.: Efficient memory management for large language model serving with PagedAttention. In: Proceedings of the ACM SIGOPS 29th Symposium on Operating Systems Principles (2023)","DOI":"10.1145\/3600006.3613165"},{"key":"16_CR17","unstructured":"Liang, P., et al.: Holistic evaluation of language models. arXiv:2211.09110 (2023)"},{"key":"16_CR18","unstructured":"Liu, J., Xia, C.S., Wang, Y., Zhang, L.: Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation. In: Advances in Neural Information Processing Systems, vol. 36 (2024)"},{"key":"16_CR19","unstructured":"Meyer, L.P., Frey, J., Brei, F., Arndt, N.: Assessing SPARQL capabilities of large language models. In: Vakaj, E., et al. (eds.) Proceedings of the 3rd International Workshop on Natural Language Processing for Knowledge Graph Creation co-located with 20th International Conference on Semantic Systems (SEMANTiCS 2024). CEUR Workshop Proceedings, vol.\u00a03874, pp. 35\u201353 (2024). https:\/\/ceur-ws.org\/Vol-3874\/paper3.pdf"},{"key":"16_CR20","unstructured":"Meyer, L.P., et al.: Developing a scalable benchmark for assessing large language models in knowledge graph engineering. In: Keshan, N., Neumaier, S., Gentile, A.L., Vahdati, S. (eds.) Proceedings of the Posters and Demo Track of the 19th International Conference on Semantic Systems (SEMANTICS 2023). CEUR Workshop Proceedings, vol.\u00a03526. CEUR-WS.org (2023). https:\/\/ceur-ws.org\/Vol-3526\/paper-04.pdf"},{"key":"16_CR21","doi-asserted-by":"publisher","unstructured":"Meyer, L.P., et al.: LLM-assisted knowledge graph engineering: Experiments with ChatGPT. In: Zinke-Wehlmann, C., Friedrich, J. (eds.) First Working Conference on Artificial Intelligence Development for a Resilient and Sustainable Tomorrow (AITomorrow) 2023, pp. 103\u2013115. Informatik aktuell (2024). https:\/\/doi.org\/10.1007\/978-3-658-43705-3_8","DOI":"10.1007\/978-3-658-43705-3_8"},{"key":"16_CR22","doi-asserted-by":"publisher","unstructured":"Mihindukulasooriya, N., Tiwari, S., Enguix, C.F., Lata, K.: Text2KGBench: a benchmark for ontology-driven knowledge graph generation from text. In: Payne, T.R., Presutti, V., Qi, G., Poveda-Villal\u00f3n, M., Stoilos, G., Hollink, L., Kaoudi, Z., Cheng, G., Li, J. (eds.) The Semantic Web \u2013 ISWC 2023, pp. 247\u2013265. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-47243-5_14","DOI":"10.1007\/978-3-031-47243-5_14"},{"key":"16_CR23","doi-asserted-by":"publisher","unstructured":"Pan, S., Luo, L., Wang, Y., Chen, C., Wang, J., Wu, X.: Unifying large language models and knowledge graphs: a roadmap. IEEE Trans. Knowl. Data Eng. (TKDE) (2024). https:\/\/doi.org\/10.1109\/TKDE.2024.3352100","DOI":"10.1109\/TKDE.2024.3352100"},{"key":"16_CR24","unstructured":"Shorten, C., et al.: Structuredrag: Json response formatting with large language models. arXiv:2408.11061 (2024)"},{"key":"16_CR25","unstructured":"Srivastava, A., et\u00a0al.: Beyond the imitation game: quantifying and extrapolating the capabilities of language models. Trans. Mach. Learn. Res. (2023)"},{"issue":"2","key":"16_CR26","doi-asserted-by":"publisher","first-page":"293","DOI":"10.3233\/sw-180312","volume":"10","author":"R Usbeck","year":"2019","unstructured":"Usbeck, R., et al.: Benchmarking question answering systems. Semant. Web 10(2), 293\u2013304 (2019). https:\/\/doi.org\/10.3233\/sw-180312","journal-title":"Semant. Web"},{"key":"16_CR27","unstructured":"Yang, A., et\u00a0al.: Qwen2 technical report. arXiv:2407.10671 (2024)"},{"key":"16_CR28","doi-asserted-by":"publisher","unstructured":"Zahera, H.M., Ali, M., Sherif, M.A., Moussallem, D., Ngonga\u00a0Ngomo, A.C.: Generating sparql from natural language using chain-of-thoughts prompting. In: SEMANTiCS 2024: Knowledge Graphs in the Age of Language Models and Neuro-Symbolic AI. Amsterdam, Netherlands (2024). https:\/\/doi.org\/10.3233\/ssw240028","DOI":"10.3233\/ssw240028"},{"key":"16_CR29","doi-asserted-by":"publisher","unstructured":"Zhu, Y., et al.: LLMs for knowledge graph construction and reasoning: recent capabilities and future opportunities. World Wide Web 27(5) (2024). https:\/\/doi.org\/10.1007\/s11280-024-01297-w","DOI":"10.1007\/s11280-024-01297-w"}],"container-title":["Lecture Notes in Computer Science","The Semantic Web"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-94578-6_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,30]],"date-time":"2025-05-30T22:02:58Z","timestamp":1748642578000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-94578-6_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031945779","9783031945786"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-94578-6_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"31 May 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors used free Google Cloud credits for the execution of Gemini models, however due to the setup and technical nature of the evaluation this has no effect on the results.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ESWC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Semantic Web Conference","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portoroz","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Slovenia","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 June 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"5 June 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"esws2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2025.eswc-conferences.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}