{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:11:04Z","timestamp":1784178664165,"version":"3.55.0"},"publisher-location":"Cham","reference-count":63,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032212993","type":"print"},{"value":"9783032213006","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-21300-6_5","type":"book-chapter","created":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T13:08:53Z","timestamp":1774357733000},"page":"70-87","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["The Effect of\u00a0Document Summarization on\u00a0LLM-Based Relevance Judgments"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-0980-6254","authenticated-orcid":false,"given":"Samaneh","family":"Mohtadi","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9191-3280","authenticated-orcid":false,"given":"Kevin","family":"Roitero","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2852-168X","authenticated-orcid":false,"given":"Stefano","family":"Mizzaro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7311-3693","authenticated-orcid":false,"given":"Gianluca","family":"Demartini","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,3,25]]},"reference":[{"key":"5_CR1","doi-asserted-by":"publisher","unstructured":"Adams, G., Fabbri, A., Ladhak, F., Lehman, E., Elhadad, N.: From sparse to dense: GPT-4 summarization with chain of density prompting. In: Proceedings of the 4th New Frontiers in Summarization Workshop, pp. 68\u201374. Association for Computational Linguistics, Singapore (2023). https:\/\/doi.org\/10.18653\/v1\/2023.newsum-1.7","DOI":"10.18653\/v1\/2023.newsum-1.7"},{"key":"5_CR2","doi-asserted-by":"publisher","unstructured":"Arabzadeh, N., Clarke, C.L.: Benchmarking LLM-based relevance judgment methods. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), pp. 3194\u20133204 (2025). https:\/\/doi.org\/10.1145\/3726302.3730305","DOI":"10.1145\/3726302.3730305"},{"key":"5_CR3","doi-asserted-by":"publisher","unstructured":"Askari, H., Chhabra, A., Chen, M., Mohapatra, P.: Assessing LLMs for zero-shot abstractive summarization through the lens of relevance paraphrasing. In: Findings of the Association for Computational Linguistics: NAACL 2025, pp. 2187\u20132201. Association for Computational Linguistics, Albuquerque, New Mexico (2025). https:\/\/doi.org\/10.18653\/v1\/2025.findings-naacl.116","DOI":"10.18653\/v1\/2025.findings-naacl.116"},{"key":"5_CR4","doi-asserted-by":"publisher","unstructured":"Bajaj, P., et al.: MS MARCO: a human generated machine reading comprehension dataset. arXiv preprint arXiv:1611.09268 (2016). https:\/\/doi.org\/10.48550\/arXiv.1611.09268","DOI":"10.48550\/arXiv.1611.09268"},{"key":"5_CR5","doi-asserted-by":"publisher","unstructured":"Balog, K., Metzler, D., Qin, Z.: Rankers, judges, and assistants: towards understanding the interplay of LLMs in information retrieval evaluation. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), pp. 3865\u20133875 (2025). https:\/\/doi.org\/10.1145\/3726302.3730348","DOI":"10.1145\/3726302.3730348"},{"key":"5_CR6","doi-asserted-by":"publisher","unstructured":"Bang, Y., Chen, D., Lee, N., Fung, P.: Measuring political bias in large language models: What is said and how it is said. In: Proceedings of the 46th European Conference on Information Retrieval (ECIR 2024), pp. xxx\u2013xxx. Springer, Glasgow (2024). https:\/\/doi.org\/10.1007\/978-3-031-56063-7_37","DOI":"10.1007\/978-3-031-56063-7_37"},{"key":"5_CR7","unstructured":"Brown, T., et\u00a0al.: Language models are few-shot learners. In: Advances in Neural Information Processing Systems (NeurIPS 2020). vol.\u00a033, pp. 1877\u20131901 (2020). https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"5_CR8","doi-asserted-by":"publisher","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proceedings of the 27th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 25\u201332. ACM (2004). https:\/\/doi.org\/10.1145\/1008992.1009000","DOI":"10.1145\/1008992.1009000"},{"key":"5_CR9","doi-asserted-by":"publisher","unstructured":"Carterette, B., Soboroff, I.: The effect of assessor error on IR system evaluation. In: Proceedings of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 539\u2013546. ACM (2010). https:\/\/doi.org\/10.1145\/1835449.1835540","DOI":"10.1145\/1835449.1835540"},{"key":"5_CR10","doi-asserted-by":"publisher","unstructured":"Clarke, C.L.A., Dietz, L.: LLM-based relevance assessment still can\u2019t replace human relevance assessment. arXiv preprint arXiv:2412.17156 (2024). https:\/\/doi.org\/10.48550\/arXiv.2412.17156","DOI":"10.48550\/arXiv.2412.17156"},{"issue":"6","key":"5_CR11","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1108\/eb050097","volume":"19","author":"C Cleverdon","year":"1967","unstructured":"Cleverdon, C.: The cranfield tests on index language devices. ASLIB Proc. 19(6), 173\u2013194 (1967). https:\/\/doi.org\/10.1108\/eb050097","journal-title":"ASLIB Proc."},{"issue":"1","key":"5_CR12","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Measur. 20(1), 37\u201346 (1960). https:\/\/doi.org\/10.1177\/001316446002000104","journal-title":"Educ. Psychol. Measur."},{"issue":"4","key":"5_CR13","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen, J.: Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit. Psychol. Bull. 70(4), 213\u2013220 (1968). https:\/\/doi.org\/10.1037\/h0026256","journal-title":"Psychol. Bull."},{"key":"5_CR14","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D.: Overview of the TREC 2020 deep learning track (2021). https:\/\/arxiv.org\/abs\/2102.07662","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"5_CR15","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M.: Overview of the TREC 2019 deep learning track (2020). https:\/\/arxiv.org\/abs\/2003.07820","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"5_CR16","unstructured":"Dietz, L., et al.: LLM-evaluation tropes: perspectives on the validity of LLM-evaluations (2025). https:\/\/arxiv.org\/abs\/2504.19076"},{"key":"5_CR17","doi-asserted-by":"publisher","unstructured":"Dubey, A., et\u00a0al.: The llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024). https:\/\/doi.org\/10.48550\/arXiv.2407.21783","DOI":"10.48550\/arXiv.2407.21783"},{"key":"5_CR18","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1162\/tacl_a_00373","volume":"9","author":"AR Fabbri","year":"2021","unstructured":"Fabbri, A.R., Kry\u015bci\u0144ski, W., McCann, B., Xiong, C., Socher, R., Radev, D.: Summeval: re-evaluating summarization evaluation. Trans. Assoc. Comput. Linguistics 9, 391\u2013409 (2021). https:\/\/doi.org\/10.1162\/tacl_a_00373","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"5_CR19","doi-asserted-by":"publisher","unstructured":"Faggioli, G., et al.: Perspectives on large language models for relevance judgment. In: Proceedings of the 2023 ACM SIGIR International Conference on the Theory of Information Retrieval (ICTIR \u201923), pp. 39\u201350 (2023). https:\/\/doi.org\/10.1145\/3578337.3605136","DOI":"10.1145\/3578337.3605136"},{"key":"5_CR20","doi-asserted-by":"publisher","unstructured":"Faggioli, G., et al.: Who determines what is relevant? humans or AI? why not both? Commun. ACM 67(4), 31\u201334 (2024). https:\/\/doi.org\/10.1145\/3624730","DOI":"10.1145\/3624730"},{"key":"5_CR21","doi-asserted-by":"publisher","unstructured":"Fr\u00f6be, M., et al.: Large language model relevance assessors agree with one another more than with human assessors. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), pp. 2858\u20132863 (2025). https:\/\/doi.org\/10.1145\/3726302.3730218","DOI":"10.1145\/3726302.3730218"},{"issue":"2","key":"5_CR22","doi-asserted-by":"publisher","first-page":"661","DOI":"10.1162\/coli_a_00561","volume":"51","author":"M Gao","year":"2025","unstructured":"Gao, M., Hu, X., Yin, X., Ruan, J., Pu, X., Wan, X.: LLM-based NLG evaluation: current status and challenges. Comput. Linguistics 51(2), 661\u2013687 (2025). https:\/\/doi.org\/10.1162\/coli_a_00561","journal-title":"Comput. Linguistics"},{"key":"5_CR23","unstructured":"Generation, T.R.A.: Trec 2024 rag umbrela qrels released. https:\/\/trec-rag.github.io\/annoucements\/umbrela-qrels\/?utm_source=chatgpt.com (2024), Accessed 03 Oct 2025"},{"key":"5_CR24","doi-asserted-by":"publisher","unstructured":"Gigant, T., Guinaudeau, C., Decombas, M., Dufaux, F.: Mitigating the impact of reference quality on evaluation of summarization systems with reference-free metrics. In: Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing (EMNLP 2024), pp. 19355\u201319368. Association for Computational Linguistics, Miami (2024). https:\/\/doi.org\/10.18653\/v1\/2024.emnlp-main.1078, https:\/\/aclanthology.org\/2024.emnlp-main.1078\/","DOI":"10.18653\/v1\/2024.emnlp-main.1078"},{"key":"5_CR25","doi-asserted-by":"publisher","unstructured":"Goyal, T., Li, J.J., Durrett, G.: News summarization and evaluation in the era of GPT-3. arXiv preprint arXiv:2209.12356 (2022). https:\/\/doi.org\/10.48550\/arXiv.2209.12356","DOI":"10.48550\/arXiv.2209.12356"},{"issue":"4","key":"5_CR26","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. 20(4), 422\u2013446 (2002). https:\/\/doi.org\/10.1145\/582415.582418","journal-title":"ACM Trans. Inf. Syst."},{"issue":"1\u20132","key":"5_CR27","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1093\/biomet\/30.1-2.81","volume":"30","author":"MG Kendall","year":"1938","unstructured":"Kendall, M.G.: A new measure of rank correlation. Biometrika 30(1\u20132), 81\u201393 (1938). https:\/\/doi.org\/10.1093\/biomet\/30.1-2.81","journal-title":"Biometrika"},{"key":"5_CR28","doi-asserted-by":"publisher","unstructured":"Krippendorff, K.: Content analysis: an introduction to its methodology. SAGE Publications, Thousand Oaks 4 edn. (2018). https:\/\/doi.org\/10.4135\/9781071878781","DOI":"10.4135\/9781071878781"},{"key":"5_CR29","unstructured":"Li, D., et al.: From generation to judgment: opportunities and challenges of LLM-as-a-judge. arXiv preprint arXiv:2411.16594 (2024). https:\/\/arxiv.org\/abs\/2411.16594"},{"key":"5_CR30","unstructured":"Li, H., et al.: LLMs-as-judges: a comprehensive survey on LLM-based evaluation methods (2024). https:\/\/arxiv.org\/abs\/2412.05579"},{"key":"5_CR31","unstructured":"Lin, C.Y.: ROUGE: a package for automatic evaluation of summaries. In: Text Summarization Branches Out: Proceedings of the ACL-04 Workshop, pp. 74\u201381. Association for Computational Linguistics, Barcelona (2004). https:\/\/aclanthology.org\/W04-1013"},{"key":"5_CR32","doi-asserted-by":"publisher","unstructured":"Liu, Y., Iter, D., Xu, Y., Wang, S., Xu, R., Zhu, C.: G-eval: NLG evaluation using GPT-4 with better human alignment. In: Bouamor, H., Pino, J., Bali, K. (eds.) Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pp. 2511\u20132522. Association for Computational Linguistics, Singapore (2023). https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.153","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"5_CR33","doi-asserted-by":"publisher","unstructured":"Liu, Y., et al.: Benchmarking generation and evaluation capabilities of large language models for instruction controllable summarization. In: Findings of the Association for Computational Linguistics: NAACL 2024, pp. 4481\u20134501 (2024). https:\/\/doi.org\/10.18653\/v1\/2024.findings-naacl.280","DOI":"10.18653\/v1\/2024.findings-naacl.280"},{"key":"5_CR34","doi-asserted-by":"publisher","unstructured":"Min, H., et al.: Towards multi-dimensional evaluation of LLM summarization across domains and languages. In: Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics, vol. 1, pp. 14417\u201314450. Association for Computational Linguistics, Vienna (2025). https:\/\/doi.org\/10.18653\/v1\/2025.acl-long.702","DOI":"10.18653\/v1\/2025.acl-long.702"},{"key":"5_CR35","doi-asserted-by":"publisher","unstructured":"Onishi, T., Caverlee, J.: Political bias of large language models in few-shot news summarization. In: Advances in Bias and Fairness in Information Retrieval: International Workshop on Algorithmic Bias in Search and Recommendation (Bias@ECIR 2024). Communications in Computer and Information Science, vol.\u00a02227, pp. 32\u201345. Springer (2024). https:\/\/doi.org\/10.1007\/978-3-031-71975-2_3","DOI":"10.1007\/978-3-031-71975-2_3"},{"key":"5_CR36","unstructured":"OpenAI: GPT-4 technical report (2023). https:\/\/arxiv.org\/abs\/2303.08774"},{"key":"5_CR37","doi-asserted-by":"publisher","unstructured":"OpenAI: Gpt-4 technical report. arXiv preprint arXiv:2303.08774 (2023). https:\/\/doi.org\/10.48550\/arXiv.2303.08774","DOI":"10.48550\/arXiv.2303.08774"},{"key":"5_CR38","unstructured":"Organizers, N.T.: Trec 2024 retrieval-augmented generation (rag) track guidelines. https:\/\/trec-rag.github.io\/annoucements\/2024-track-guidelines\/ (2024), Accessed 27 Sept 2025"},{"key":"5_CR39","unstructured":"Page, L., Brin, S., Motwani, R., Winograd, T.: The pagerank citation ranking: bringing order to the web. Tech. Rep. 1999-66, Stanford InfoLab (1999). http:\/\/ilpubs.stanford.edu:8090\/422\/, previous number = SIDL-WP-1999-0120"},{"key":"5_CR40","doi-asserted-by":"publisher","unstructured":"Pan, B., Hembrooke, H., Joachims, T., Lorigo, L., Gay, G., Granka, L.: In google we trust: Users\u2019 decisions on rank, position, and relevance. J. Comput. Mediated Commun. 12(3), 801\u2013823 (2007). https:\/\/doi.org\/10.1111\/j.1083-6101.2007.00351.x","DOI":"10.1111\/j.1083-6101.2007.00351.x"},{"key":"5_CR41","doi-asserted-by":"publisher","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: BLEU: a method for automatic evaluation of machine translation. In: Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (ACL), pp. 311\u2013318. Association for Computational Linguistics, Philadelphia (2002). https:\/\/doi.org\/10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"key":"5_CR42","doi-asserted-by":"publisher","unstructured":"Peters, U., Chin-Yee, B.: Generalization bias in large language model summarization of scientific research. Royal Soc. Open Sci. 12(4), 241776 (2025). https:\/\/doi.org\/10.1098\/rsos.241776","DOI":"10.1098\/rsos.241776"},{"key":"5_CR43","unstructured":"Rahmani, H.A., et al.: Judging the judges: a collection of LLM-generated relevance judgements. arXiv preprint arXiv:2502.13908 (2025). https:\/\/arxiv.org\/abs\/2502.13908"},{"key":"5_CR44","doi-asserted-by":"crossref","unstructured":"Rehman, T., Ghosh, S., Das, K., Bhattacharjee, S., Sanyal, D.K., Chattopadhyay, S.: Evaluating LLMs and pre-trained models for text summarization across diverse datasets (2025). https:\/\/arxiv.org\/abs\/2502.19339","DOI":"10.1201\/9781003675235-41"},{"key":"5_CR45","doi-asserted-by":"publisher","unstructured":"Roitero, K., Wright, D., Soprano, M., Augenstein, I., Mizzaro, S.: Efficiency and effectiveness of LLM-based summarization of evidence in crowdsourced fact-checking. In: Proceedings of the the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), pp. 457\u2013467 (2025). https:\/\/doi.org\/10.1145\/3726302.3729960","DOI":"10.1145\/3726302.3729960"},{"issue":"5","key":"5_CR46","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Inf. Retrieval 11(5), 447\u2013470 (2008). https:\/\/doi.org\/10.1007\/s10791-008-9059-7","journal-title":"Inf. Retrieval"},{"key":"5_CR47","doi-asserted-by":"publisher","unstructured":"Soboroff, I.: Don\u2019t use LLMs to make relevance judgments. Inf. Retrieval Res. J. (2024). https:\/\/doi.org\/10.54195\/irrj.19625","DOI":"10.54195\/irrj.19625"},{"key":"5_CR48","doi-asserted-by":"publisher","unstructured":"Soboroff, I.: Don\u2019t use LLMs to make relevance judgments. Inf. Retrieval Res. 1(1), 29\u201346 (2025). https:\/\/doi.org\/10.54195\/irrj.19625","DOI":"10.54195\/irrj.19625"},{"key":"5_CR49","doi-asserted-by":"publisher","unstructured":"Song, H., Su, H., Shalyminov, I., Cai, J., Mansour, S.: Finesure: fine-grained summarization evaluation using LLMs. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics, vol. 1, pp. 906\u2013922. Association for Computational Linguistics, Bangkok (2024). https:\/\/doi.org\/10.18653\/v1\/2024.acl-long.51","DOI":"10.18653\/v1\/2024.acl-long.51"},{"issue":"3\/4","key":"5_CR50","doi-asserted-by":"publisher","first-page":"441","DOI":"10.2307\/1422689","volume":"100","author":"C Spearman","year":"1987","unstructured":"Spearman, C.: The proof and measurement of association between two things. Am. J. Psychol. 100(3\/4), 441\u2013471 (1987). https:\/\/doi.org\/10.2307\/1422689","journal-title":"Am. J. Psychol."},{"key":"5_CR51","doi-asserted-by":"publisher","unstructured":"Takehi, R., Voorhees, E.M., Sakai, T., Soboroff, I.: LLM-assisted relevance assessments: When should we ask LLMs for help? In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), pp. 95\u2013105 (2025). https:\/\/doi.org\/10.1145\/3726302.3729916","DOI":"10.1145\/3726302.3729916"},{"key":"5_CR52","doi-asserted-by":"publisher","unstructured":"Thomas, P., Spielman, S., Craswell, N., Mitra, B.: Large language models can accurately predict searcher preferences. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201924), pp. 1930\u20131940 (2024). https:\/\/doi.org\/10.1145\/3626772.3657707","DOI":"10.1145\/3626772.3657707"},{"key":"5_CR53","doi-asserted-by":"publisher","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Campos, D., Craswell, N., Soboroff, I., Lin, J.: A large-scale study of relevance assessments with large language models using UMBRELA. In: Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR \u201925), pp. 358\u2013368 (2025). https:\/\/doi.org\/10.1145\/3731120.3744605","DOI":"10.1145\/3731120.3744605"},{"key":"5_CR54","doi-asserted-by":"publisher","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Craswell, N., Lin, J.: Umbrela: Umbrela is the (open-source reproduction of the) bing relevance assessor. arXiv preprint arXiv:2406.06519 (2024). https:\/\/doi.org\/10.48550\/arXiv.2406.06519","DOI":"10.48550\/arXiv.2406.06519"},{"key":"5_CR55","doi-asserted-by":"publisher","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. In: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 315\u2013323. ACM, New York (1998). https:\/\/doi.org\/10.1145\/290941.291011","DOI":"10.1145\/290941.291011"},{"key":"5_CR56","doi-asserted-by":"publisher","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manag. 36(5), 697\u2013716 (2000). https:\/\/doi.org\/10.1016\/S0306-4573(00)00010-8, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0306457300000108","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"5_CR57","doi-asserted-by":"publisher","unstructured":"Wan, D., Vig, J., Bansal, M., Joty, S.: On positional bias of faithfulness for long-form summarization. In: Proceedings of the 2025 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, vol. 1, pp. 8791\u20138810 (2025). https:\/\/doi.org\/10.18653\/v1\/2025.naacl-long.442, https:\/\/aclanthology.org\/2025.naacl-long.442\/","DOI":"10.18653\/v1\/2025.naacl-long.442"},{"key":"5_CR58","unstructured":"Webber, W., Moffat, A., Zobel, J.: The effect of pooling and evaluation depth on metric stability. In: Sakai, T., Sanderson, M., Webber, W. (eds.) Proceedings of the 3rd International Workshop on Evaluating Information Access (EVIA@NTCIR), pp. 7\u201315. National Institute of Informatics (NII), Tokyo (2010). http:\/\/research.nii.ac.jp\/ntcir\/workshop\/OnlineProceedings8\/EVIA\/03-EVIA2010-WebberW.pdf"},{"key":"5_CR59","doi-asserted-by":"publisher","unstructured":"Webber, W., Moffat, A., Zobel, J.: A similarity measure for indefinite rankings. ACM Trans. Inf. Syst. 28(4), 20:1\u201320:38 (2010). https:\/\/doi.org\/10.1145\/1852102.1852106","DOI":"10.1145\/1852102.1852106"},{"key":"5_CR60","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K.Q., Artzi, Y.: BERTScore: evaluating text generation with BERT. In: Proceedings of the 8th International Conference on Learning Representations (ICLR) (2020). https:\/\/openreview.net\/forum?id=SkeHuCVFDr"},{"key":"5_CR61","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1162\/tacl_a_00632","volume":"12","author":"T Zhang","year":"2024","unstructured":"Zhang, T., Ladhak, F., Durmus, E., Liang, P., McKeown, K., Hashimoto, T.B.: Benchmarking large language models for news summarization. Trans. Assoc. Comput. Linguistics 12, 39\u201357 (2024). https:\/\/doi.org\/10.1162\/tacl_a_00632","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"5_CR62","doi-asserted-by":"publisher","unstructured":"Zhuang, H., Qin, Z., Hui, K., Wu, J., Yan, L., Wang, X., Bendersky, M.: Beyond yes and no: improving zero-shot LLM rankers via scoring fine-grained relevance labels. In: Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, vol. 2, pp. 358\u2013370. Association for Computational Linguistics, Mexico City (2024). https:\/\/doi.org\/10.18653\/v1\/2024.naacl-short.31","DOI":"10.18653\/v1\/2024.naacl-short.31"},{"key":"5_CR63","doi-asserted-by":"publisher","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 307\u2013314. ACM, New York (1998). https:\/\/doi.org\/10.1145\/290941.291015","DOI":"10.1145\/290941.291015"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-21300-6_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T13:09:03Z","timestamp":1774357743000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-21300-6_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032212993","9783032213006"],"references-count":63,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-21300-6_5","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"25 March 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Delft","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 March 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"48","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2026.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}