{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,21]],"date-time":"2026-09-21T02:08:59Z","timestamp":1789956539430,"version":"4.0.1"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031719073","type":"print"},{"value":"9783031719080","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-71908-0_3","type":"book-chapter","created":{"date-parts":[[2024,9,19]],"date-time":"2024-09-19T16:29:00Z","timestamp":1726763340000},"page":"53-72","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Overview of\u00a0ELOQUENT 2024\u2014Shared Tasks for\u00a0Evaluating Generative Language Model Quality"],"prefix":"10.1007","author":[{"given":"Jussi","family":"Karlgren","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luise","family":"D\u00fcrlich","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Evangelia","family":"Gogoulou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liane","family":"Guillou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joakim","family":"Nivre","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Magnus","family":"Sahlgren","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aarne","family":"Talman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shorouq","family":"Zahra","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,19]]},"reference":[{"key":"3_CR1","doi-asserted-by":"crossref","unstructured":"Altinisik, E., Sajjad, H., Sencar, H.T., Messaoud, S., Chawla, S.: Impact of adversarial training on robustness and generalizability of language models. arXiv preprint arXiv:2211.05523 (2023)","DOI":"10.18653\/v1\/2023.findings-acl.496"},{"key":"3_CR2","unstructured":"Amrhein, C., Moghe, N., Guillou, L.: ACES: translation accuracy challenge sets for evaluating machine translation metrics. In: Proceedings of the Seventh Conference on Machine Translation (WMT), pp. 479\u2013513. Association for Computational Linguistics (2022). https:\/\/aclanthology.org\/2022.wmt-1.44"},{"key":"3_CR3","doi-asserted-by":"crossref","unstructured":"Bell, A.: Language style as audience design. Lang. Soc. 13(2) (1984)","DOI":"10.1017\/S004740450001037X"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Berdicevskis, A., et al.: SuperLim: a Swedish language understanding evaluation benchmark, pp. 8137\u20138153. Association for Computational Linguistics, Singapore (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.506"},{"key":"3_CR5","doi-asserted-by":"crossref","unstructured":"Bevendorff, J., et al.: Overview of PAN 2024: multi-author writing style analysis, multilingual text detoxification, oppositional thinking analysis, and generative ai authorship verification. In: Advances in Information Retrieval: 46th European Conference on IR Research (ECIR) (2024)","DOI":"10.1007\/978-3-031-56072-9_1"},{"key":"3_CR6","unstructured":"Bevendorff, J., et al.: Overview of the Voight-Kampff generative AI authorship verification task at PAN and ELOQUENT 2024. In: Faggioli, G., Ferro, N., Vlachos, M., Galu\u0161\u010d\u00e1kov\u00e1, P., de\u00a0Herrera, A.G.S. (eds.) Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum. CEUR-WS.org (2024)"},{"key":"3_CR7","unstructured":"Bui, A.T., et al.: The two sides of the coin: Hallucination generation and detection with evaluators for LLMs. In: Faggioli, G., Ferro, N., Galu\u0161\u010d\u00e1kov\u00e1, P., Garc\u00eda Seco\u00a0de Herrera, A. (eds.) Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum (2024)"},{"key":"3_CR8","unstructured":"Ekgren, A., et al.: GPT-SW3: An autoregressive language model for the Scandinavian languages. In: Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024), Torino, Italia (2024)"},{"key":"3_CR9","unstructured":"Ekgren, A., et al.: Lessons learned from GPT-SW3: building the first large-scale generative language model for Swedish. In: Proceedings of the Thirteenth Language Resources and Evaluation Conference (LREC) (2022)"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Faggioli, G., et\u00a0al.: Perspectives on large language models for relevance judgment. In: Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval (2023)","DOI":"10.1145\/3578337.3605136"},{"key":"3_CR11","unstructured":"Gao, Y., et al.: Retrieval-augmented generation for large language models: a survey. arXiv preprint arXiv:2312.10997 (2023)"},{"key":"3_CR12","unstructured":"Jiang, A.Q., et\u00a0al.: Mistral 7B. arXiv preprint arXiv:2310.06825 (2023)"},{"key":"3_CR13","unstructured":"Jiang, A.Q., et\u00a0al.: Mixtral of experts. arXiv preprint arXiv:2401.04088 (2024)"},{"key":"3_CR14","unstructured":"Kanerva, J., et al.: Finnish paraphrase corpus. In: Proceedings of the 23rd Nordic Conference on Computational Linguistics (NoDaLiDa), pp. 288\u2013298. Link\u00f6ping University Electronic Press, Sweden, Reykjavik, Iceland (2021)"},{"key":"3_CR15","doi-asserted-by":"publisher","unstructured":"Laurer, M., van Atteveldt, W., Casas, A., Welbers, K.: Building efficient universal classifiers with natural language inference (2023). https:\/\/doi.org\/10.48550\/arXiv.2312.17543, http:\/\/arxiv.org\/abs\/2312.17543, arXiv:2312.17543 [cs]","DOI":"10.48550\/arXiv.2312.17543"},{"key":"3_CR16","unstructured":"Luukkonen, R., et al.: Poro 34B and the blessing of multilinguality. arXiv preprint arXiv:2404.01856 (2024)"},{"key":"3_CR17","doi-asserted-by":"crossref","unstructured":"Manakul, P., Liusie, A., Gales, M.J.F.: SelfCheckGPT: zero-resource black-box hallucination detection for generative large language models. arXiv preprint arXiv arXiv:2303.08896 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.557"},{"key":"3_CR18","doi-asserted-by":"crossref","unstructured":"Mickus, T., et al.: SemEval-2024 shared task 6: shroom, a shared-task on hallucinations and related observable overgeneration mistakes. arXiv preprint arXiv:2403.07726 (2024)","DOI":"10.18653\/v1\/2024.semeval-1.273"},{"key":"3_CR19","doi-asserted-by":"crossref","unstructured":"Moradi, M., Samwald, M.: Evaluating the robustness of neural language models to input perturbations. In: Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (2021)","DOI":"10.18653\/v1\/2021.emnlp-main.117"},{"key":"3_CR20","unstructured":"Neralla, V., Bijl\u00a0de Vroe, S.: Evaluating Poro-34B-chat and mistral-7B-instruct-v0.1: LLM system description for ELOQUENT at CLEF 2024. In: Faggioli, G., Ferro, N., Vlachos, M., Galu\u0161\u010d\u00e1kov\u00e1, P., de\u00a0Herrera, A.G.S. (eds.) Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum. CEUR-WS.org (2024)"},{"key":"3_CR21","unstructured":"Nielsen, D.S.: ScandiNLI: natural language inference for the Scandinavian languages (2022). https:\/\/github.com\/alexandrainst\/ScandiNLI, https:\/\/aclanthology.org\/D19-1382"},{"key":"3_CR22","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/978-3-031-42448-9_11","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"AM Sarvazyan","year":"2023","unstructured":"Sarvazyan, A.M., Gonz\u00e1lez, J.\u00c1., Rosso, P., Franco-Salvador, M.: Supervised machine-generated text detectors: family and scale matters. In: Arampatzis, A., et al. (eds.) CLEF 2023. LNCS, vol. 14163, pp. 121\u2013132. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-42448-9_11"},{"key":"3_CR23","unstructured":"Saunders, W., et al.: Self-critiquing models for assisting human evaluators. arXiv preprint arXiv:2206.05802 (2022)"},{"key":"3_CR24","unstructured":"Siino, M., Tinnirello, I.: GPT hallucination detection through prompt engineering. In: Faggioli, G., Ferro, N., Galu\u0161\u010d\u00e1kov\u00e1, P., Garc\u00eda Seco\u00a0de Herrera, A. (eds.) Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum (2024)"},{"key":"3_CR25","unstructured":"Simonsen, A.: Eloquent robustness experiment report. In: Faggioli, G., Ferro, N., Vlachos, M., Galu\u0161\u010d\u00e1kov\u00e1, P., de\u00a0Herrera, A.G.S. (eds.) Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum. CEUR-WS.org (2024)"},{"key":"3_CR26","doi-asserted-by":"crossref","unstructured":"Singhal, K., et al.: Large language models encode clinical knowledge. Nature 620(7972) (2023)","DOI":"10.1038\/s41586-023-06291-2"},{"key":"3_CR27","unstructured":"Touvron, H., et al.: Llama 2: open foundation and fine-tuned chat models (2023)"},{"key":"3_CR28","unstructured":"Wang, B., et al.: InfoBERT: improving robustness of language models from an information theoretic perspective. In: International Conference on Learning Representations (2021)"},{"key":"3_CR29","unstructured":"Wu, S., et al.: BloombergGPT: a large language model for finance. arXiv preprint arXiv:2303.17564 (2023)"},{"key":"3_CR30","doi-asserted-by":"publisher","unstructured":"Yang, Y., Zhang, Y., Tar, C., Baldridge, J.: PAWS-X: a cross-lingual adversarial dataset for paraphrase identification. In: Inui, K., Jiang, J., Ng, V., Wan, X. (eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP). pp. 3687\u20133692. Association for Computational Linguistics, Hong Kong (2019).https:\/\/doi.org\/10.18653\/v1\/D19-1382, https:\/\/aclanthology.org\/D19-1382","DOI":"10.18653\/v1\/D19-1382"},{"key":"3_CR31","unstructured":"Zheng, C., Zhou, H., Meng, F., Zhou, J., Huang, M.: Large language models are not robust multiple choice selectors. arXiv preprint arXiv:2309.03882 (2023)"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-71908-0_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,19]],"date-time":"2024-09-19T16:31:13Z","timestamp":1726763473000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-71908-0_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031719073","9783031719080"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-71908-0_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"19 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CLEF","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference of the Cross-Language Evaluation Forum for European Languages","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Grenoble","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"clef2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/clef2024.clef-initiative.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}