{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T02:36:25Z","timestamp":1783737385007,"version":"3.55.0"},"publisher-location":"Cham","reference-count":39,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032212993","type":"print"},{"value":"9783032213006","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-21300-6_6","type":"book-chapter","created":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T13:11:20Z","timestamp":1774357880000},"page":"88-103","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Query\u2013Document Dense Vectors for\u00a0LLM Relevance Judgment Bias Analysis"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-0980-6254","authenticated-orcid":false,"given":"Samaneh","family":"Mohtadi","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7311-3693","authenticated-orcid":false,"given":"Gianluca","family":"Demartini","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,3,25]]},"reference":[{"key":"6_CR1","doi-asserted-by":"publisher","unstructured":"Alaofi, M., Thomas, P., Scholer, F., Sanderson, M.: LLMs can be fooled into labelling a document as relevant: best Caf\u00e9 near me; this paper is perfectly relevant. In: Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region (SIGIR-AP 2024), pp. 32\u201341 (2024). https:\/\/doi.org\/10.1145\/3673791.3698431","DOI":"10.1145\/3673791.3698431"},{"key":"6_CR2","doi-asserted-by":"publisher","unstructured":"Altman, D.G.: Measurement in medicine: the analysis of method comparison studies. J. Roy. Stat. Soc. Ser. D (Stat.) 32, 307\u2013317 (1983). https:\/\/doi.org\/10.2307\/2987937","DOI":"10.2307\/2987937"},{"key":"6_CR3","doi-asserted-by":"publisher","unstructured":"Arabzadeh, N., Clarke, C.L.: Benchmarking LLM-based relevance judgment methods. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2025), pp. 3194\u20133204 (2025). https:\/\/doi.org\/10.1145\/3726302.3730305","DOI":"10.1145\/3726302.3730305"},{"key":"6_CR4","unstructured":"Arbabi, A., Kerschbaum, F.: Relative bias: a comparative framework for quantifying bias in LLMs (2025). https:\/\/arxiv.org\/abs\/2505.17131"},{"key":"6_CR5","unstructured":"Bajaj, P., et\u00a0al.: MS MARCO: a human generated machine reading comprehension dataset (2016). https:\/\/arxiv.org\/abs\/1611.09268"},{"key":"6_CR6","doi-asserted-by":"publisher","unstructured":"Balog, K., Metzler, D., Qin, Z.: Rankers, judges, and assistants: towards understanding the interplay of LLMs in information retrieval evaluation. In: Proceedings of the the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2025), pp. 3865\u20133875 (2025). https:\/\/doi.org\/10.1145\/3726302.3730348","DOI":"10.1145\/3726302.3730348"},{"key":"6_CR7","doi-asserted-by":"publisher","unstructured":"Bauer, C., et\u00a0al.: Frontiers of information access experimentation for research and education (Dagstuhl seminar 23031). Dagstuhl Rep. 13(1), 68\u201391 (2023). https:\/\/doi.org\/10.4230\/DagRep.13.1.68","DOI":"10.4230\/DagRep.13.1.68"},{"key":"6_CR8","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"160","DOI":"10.1007\/978-3-642-37456-2_14","volume-title":"Advances in Knowledge Discovery and Data Mining","author":"RJGB Campello","year":"2013","unstructured":"Campello, R.J.G.B., Moulavi, D., Sander, J.: Density-based clustering based on hierarchical density estimates. In: Pei, J., Tseng, V.S., Cao, L., Motoda, H., Xu, G. (eds.) PAKDD 2013. LNCS (LNAI), vol. 7819, pp. 160\u2013172. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-37456-2_14"},{"key":"6_CR9","doi-asserted-by":"publisher","unstructured":"Cicchetti, D.V., Feinstein, A.R.: High agreement but low kappa: I. The problems of two paradoxes. J. Clin. Epidemiol. 43(6), 543\u2013549 (1990). https:\/\/doi.org\/10.1016\/0895-4356(90)90158-L","DOI":"10.1016\/0895-4356(90)90158-L"},{"key":"6_CR10","unstructured":"Clarke, C.L.A., Dietz, L.: LLM-based relevance assessment still can\u2019t replace human relevance assessment. arXiv preprint arXiv:2412.17156 (2024)"},{"key":"6_CR11","doi-asserted-by":"publisher","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Meas. 20(1), 37\u201346 (1960). https:\/\/doi.org\/10.1177\/001316446002000104","DOI":"10.1177\/001316446002000104"},{"key":"6_CR12","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D.: Overview of the TREC 2020 deep learning track (2021). https:\/\/arxiv.org\/abs\/2102.07662","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"6_CR13","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M.: Overview of the TREC 2019 deep learning track (2020). https:\/\/arxiv.org\/abs\/2003.07820","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"6_CR14","doi-asserted-by":"publisher","unstructured":"Cuadra, C.A., Katter, R.V.: Opening the black box of \u2018relevance\u2019. J. Doc. 23(4), 291\u2013303 (1967). https:\/\/doi.org\/10.1108\/eb026436","DOI":"10.1108\/eb026436"},{"key":"6_CR15","doi-asserted-by":"publisher","unstructured":"Damessie, T.T., Nghiem, T.P., Scholer, F., Culpepper, J.S.: Gauging the quality of relevance assessments using inter-rater agreement. In: Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2017), pp. 1089\u20131092. Association for Computing Machinery (2017). https:\/\/doi.org\/10.1145\/3077136.3080661","DOI":"10.1145\/3077136.3080661"},{"key":"6_CR16","unstructured":"Dietz, L., et al.: LLM-evaluation tropes: perspectives on the validity of LLM-evaluations (2025). https:\/\/arxiv.org\/abs\/2504.19076"},{"key":"6_CR17","doi-asserted-by":"publisher","unstructured":"Faggioli, G., et al.: Perspectives on large language models for relevance judgment. In: Proceedings of the 2023 ACM SIGIR International Conference on the Theory of Information Retrieval (ICTIR 2023), pp. 39\u201350 (2023). https:\/\/doi.org\/10.1145\/3578337.3605136","DOI":"10.1145\/3578337.3605136"},{"key":"6_CR18","doi-asserted-by":"publisher","unstructured":"Faggioli, G., et al.: Who determines what is relevant? Humans or AI? Why not both? Commun. ACM 67(4), 31\u201334 (2024). https:\/\/doi.org\/10.1145\/3624730","DOI":"10.1145\/3624730"},{"key":"6_CR19","doi-asserted-by":"publisher","unstructured":"Fr\u00f6be, M., et al.: Large language model relevance assessors agree with one another more than with human assessors. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2025), pp. 2858\u20132863 (2025). https:\/\/doi.org\/10.1145\/3726302.3730218","DOI":"10.1145\/3726302.3730218"},{"key":"6_CR20","doi-asserted-by":"publisher","unstructured":"Gwet, K.L.: Computing inter-rater reliability and its variance in the presence of high agreement. Br. J. Math. Stat. Psychol. 61(1), 29\u201348 (2008). https:\/\/doi.org\/10.1348\/000711006X126600","DOI":"10.1348\/000711006X126600"},{"key":"6_CR21","unstructured":"Haley, D.T., Thomas, P., Petre, M., De\u00a0Roeck, A.: Using a new inter-rater reliability statistic. Technical report. Technical Report 2008\/15, Department of Computing, The Open University, Milton Keynes, UK (2008). https:\/\/oro.open.ac.uk\/90234\/1\/TR2008_15.pdf"},{"key":"6_CR22","unstructured":"Li, D., et\u00a0al.: From generation to judgment: opportunities and challenges of LLM-as-a-judge (2024). https:\/\/arxiv.org\/abs\/2411.16594"},{"key":"6_CR23","unstructured":"Li, H., et al.: LLMs-as-judges: a comprehensive survey on LLM-based evaluation methods (2024). https:\/\/arxiv.org\/abs\/2412.05579"},{"key":"6_CR24","doi-asserted-by":"publisher","unstructured":"Minozzi, S., Cinquini, M., Gianola, S., Gonzalez-Lorenzo, M., Banzi, R.: Kappa and AC1\/2 statistics: beyond the paradox. J. Clin. Epidemiol. 142, 328\u2013329 (2022). https:\/\/doi.org\/10.1016\/j.jclinepi.2021.12.004","DOI":"10.1016\/j.jclinepi.2021.12.004"},{"key":"6_CR25","unstructured":"Page, L., Brin, S., Motwani, R., Winograd, T.: The PageRank citation ranking: bringing order to the web. Technical report, 1999-66, Stanford InfoLab, November 1999. http:\/\/ilpubs.stanford.edu:8090\/422\/. Previous number = SIDL-WP-1999-0120"},{"issue":"3","key":"6_CR26","doi-asserted-by":"publisher","first-page":"801","DOI":"10.1111\/j.1083-6101.2007.00351.x","volume":"12","author":"B Pan","year":"2007","unstructured":"Pan, B., Hembrooke, H., Joachims, T., Lorigo, L., Gay, G., Granka, L.: In google we trust: users\u2019 decisions on rank, position, and relevance. J. Comput.-Mediat. Commun. 12(3), 801\u2013823 (2007). https:\/\/doi.org\/10.1111\/j.1083-6101.2007.00351.x","journal-title":"J. Comput.-Mediat. Commun."},{"key":"6_CR27","doi-asserted-by":"crossref","unstructured":"Rahmani, H.A., Ramineni, V., Craswell, N., Mitra, B., Yilmaz, E.: Towards understanding bias in synthetic data for evaluation. arXiv preprint (2025). https:\/\/arxiv.org\/abs\/2506.10301","DOI":"10.1145\/3746252.3760908"},{"key":"6_CR28","unstructured":"Rees, A.M., Schultz, D.G.: A field experimental approach to the study of relevance assessments in relation to document searching: final report to the national science foundation. Volume ii, appendices. Technical report, ED027910, ERIC (1967). https:\/\/eric.ed.gov\/?id=ED027910"},{"key":"6_CR29","doi-asserted-by":"publisher","unstructured":"Silveira, P.S.P., Siqueira, J.O.: Better to be in agreement than in bad company: a critical analysis of many kappa-like tests. Behav. Res. Methods 55(7), 3326\u20133347 (2023). https:\/\/doi.org\/10.3758\/s13428-023-02124-5","DOI":"10.3758\/s13428-023-02124-5"},{"key":"6_CR30","doi-asserted-by":"publisher","unstructured":"Soboroff, I.: Don\u2019t use LLMs to make relevance judgments. Inf. Retrieval Res. 1(1), 29\u201346 (2025). https:\/\/doi.org\/10.54195\/irrj.19625","DOI":"10.54195\/irrj.19625"},{"key":"6_CR31","doi-asserted-by":"publisher","unstructured":"Su, H., et al.: One embedder, any task: instruction-finetuned text embeddings. In: Rogers, A., Boyd-Graber, J., Okazaki, N. (eds.) Findings of the Association for Computational Linguistics: ACL 2023, pp. 1102\u20131121. Association for Computational Linguistics, Toronto, Canada, July 2023. https:\/\/doi.org\/10.18653\/v1\/2023.findings-acl.71","DOI":"10.18653\/v1\/2023.findings-acl.71"},{"key":"6_CR32","doi-asserted-by":"publisher","unstructured":"Thomas, P., Spielman, S., Craswell, N., Mitra, B.: Large language models can accurately predict searcher preferences. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2024), pp. 1930\u20131940 (2024). https:\/\/doi.org\/10.1145\/3626772.3657707","DOI":"10.1145\/3626772.3657707"},{"key":"6_CR33","doi-asserted-by":"publisher","unstructured":"Upadhyay, S., et al.: A large-scale study of relevance assessments with large language models using UMBRELA. In: Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR 2025), pp. 358\u2013368 (2025). https:\/\/doi.org\/10.1145\/3731120.3744605","DOI":"10.1145\/3731120.3744605"},{"key":"6_CR34","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Craswell, N., Lin, J.: UMBRELA: UMbrela is the (open-source reproduction of the) Bing relevance assessor (2024). https:\/\/arxiv.org\/abs\/2406.06519"},{"key":"6_CR35","doi-asserted-by":"publisher","unstructured":"Vidgen, B., Thrush, T., Waseem, Z., Kiela, D.: Learning from the worst: dynamically generated datasets to improve online hate detection. In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics (ACL 2021), pp. 1667\u20131682. Association for Computational Linguistics (2021). https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.132. https:\/\/arxiv.org\/pdf\/2106.09775","DOI":"10.18653\/v1\/2021.acl-long.132"},{"key":"6_CR36","unstructured":"Wang, L., et al.: Text embeddings by weakly-supervised contrastive pre-training (2022). https:\/\/arxiv.org\/abs\/2212.03533"},{"key":"6_CR37","doi-asserted-by":"publisher","unstructured":"Wongpakaran, N., Wongpakaran, T., Wedding, D., Gwet, K.L.: A comparison of Cohen\u2019s Kappa and Gwet\u2019s AC1 when calculating inter-rater reliability coefficients: a study conducted with personality disorder samples. BMC Med. Res. Methodol. 13(1), 61 (2013). https:\/\/doi.org\/10.1186\/1471-2288-13-61","DOI":"10.1186\/1471-2288-13-61"},{"key":"6_CR38","unstructured":"Zhang, Y., et\u00a0al.: Qwen3 embedding: advancing text embedding and reranking through foundation models (2025). https:\/\/arxiv.org\/abs\/2506.05176"},{"key":"6_CR39","doi-asserted-by":"publisher","unstructured":"Zhuang, H., et al.: Beyond yes and no: improving zero-shot LLM rankers via scoring fine-grained relevance labels. In: Proceedings of NAACL-HLT 2024 (Short Papers), pp. 358\u2013370 (2024). https:\/\/doi.org\/10.18653\/v1\/2024.naacl-short.31. https:\/\/aclanthology.org\/2024.naacl-short.31\/","DOI":"10.18653\/v1\/2024.naacl-short.31"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-21300-6_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T13:11:32Z","timestamp":1774357892000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-21300-6_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032212993","9783032213006"],"references-count":39,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-21300-6_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"25 March 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Delft","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 March 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"48","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2026.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}