{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:17:35Z","timestamp":1784179055891,"version":"3.55.0"},"publisher-location":"Cham","reference-count":40,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032212887","type":"print"},{"value":"9783032212894","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-21289-4_37","type":"book-chapter","created":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T01:06:18Z","timestamp":1774314378000},"page":"582-597","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Insider Knowledge: How Much Can RAG Systems Gain from\u00a0Evaluation Secrets?"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1624-3907","authenticated-orcid":false,"given":"Laura","family":"Dietz","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5779-1662","authenticated-orcid":false,"given":"Bryan","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9931-2861","authenticated-orcid":false,"given":"William","family":"Walden","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,3,25]]},"reference":[{"key":"37_CR1","doi-asserted-by":"publisher","unstructured":"Abbasiantaeb, Z., Lupart, S., Azzopardi, L., Dalton, J., Aliannejadi, M.: Conversational gold: evaluating personalized conversational search system using gold nuggets. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 3455\u20133465 (2025). https:\/\/doi.org\/10.1145\/3726302.3730316","DOI":"10.1145\/3726302.3730316"},{"key":"37_CR2","doi-asserted-by":"crossref","unstructured":"Alaofi, M., Arabzadeh, N., Clarke, C.L., Sanderson, M.: Generative information retrieval evaluation. In: Information access in the era of generative AI, pp. 135\u2013159, Springer (2024)","DOI":"10.1007\/978-3-031-73147-1_6"},{"key":"37_CR3","doi-asserted-by":"publisher","unstructured":"Arabzadeh, N., Clarke, C.L.: Benchmarking LLM-based relevance judgment methods. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 3194\u20133204 (2025). https:\/\/doi.org\/10.1145\/3726302.3744382","DOI":"10.1145\/3726302.3744382"},{"key":"37_CR4","doi-asserted-by":"publisher","unstructured":"Arabzadeh, N., Clarke, C.L.A.: A human-AI comparative analysis of prompt sensitivity in LLM-based relevance judgment. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), ACM (2025). https:\/\/doi.org\/10.1145\/3726302.3730374","DOI":"10.1145\/3726302.3730374"},{"key":"37_CR5","doi-asserted-by":"crossref","unstructured":"Balog, K., Metzler, D., Qin, Z.: Rankers, judges, and assistants: towards understanding the interplay of LLMs in information retrieval evaluation. arXiv preprint arXiv:2503.19092 (2025)","DOI":"10.1145\/3726302.3730348"},{"key":"37_CR6","unstructured":"Bardas, N., Mordo, T., Kurland, O., Tennenholtz, M., Zur, G.: Prompt-based document modifications in ranking competitions. arXiv preprint arXiv:2502.07315 (2025)"},{"key":"37_CR7","doi-asserted-by":"crossref","unstructured":"Ben\u00a0Basat, R., Tennenholtz, M., Kurland, O.: The probability ranking principle is not optimal in adversarial retrieval settings. In: Proceedings of the 2015 International Conference on The Theory of Information Retrieval, pp. 51\u201360, ICTIR \u201915 (2015)","DOI":"10.1145\/2808194.2809456"},{"key":"37_CR8","unstructured":"Borgeaud, S., et\u00a0al.: Improving language models by retrieving from trillions of tokens. In: International Conference on Machine Learning, pp. 2206\u20132240, PMLR (2022)"},{"key":"37_CR9","doi-asserted-by":"publisher","unstructured":"Clarke, C.L.A., Dietz, L.: LLM-based relevance assessment still can\u2019t replace human relevance assessment. In: EVIA 2025: Proceedings of the Tenth International Workshop on Evaluating Information Access (EVIA 2025), a Satellite Workshop of the NTCIR-18 Conference, June 10-13, 2025, Tokyo, Japan, pp. 1\u20135 (2025). https:\/\/doi.org\/10.20736\/0002002105","DOI":"10.20736\/0002002105"},{"key":"37_CR10","doi-asserted-by":"crossref","unstructured":"Dhol\u00e9, K., Chandradevan, R., Agichtein, E.: AdvERSEM: Adversarial robustness testing and training of LLM-based groundedness evaluators via manipulation of semantic structures. In: 14th Joint Conference on Lexical and Computational Semantics (2025)","DOI":"10.18653\/v1\/2025.starsem-1.32"},{"key":"37_CR11","doi-asserted-by":"crossref","unstructured":"Dietz, L.: A workbench for autograding retrieve\/generate systems. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1963\u20131972 (2024)","DOI":"10.1145\/3626772.3657871"},{"key":"37_CR12","unstructured":"Dietz, L., et al.: Incorporating Q&A nuggets into retrieval-augmented generation. In: Proceedings of the 48th European Conference on Information Retrieval (ECIR 2026) (2026)"},{"key":"37_CR13","unstructured":"Dietz, L., Li, B., Mayfield, J., Lawrie, D., Yang, E., Walden, W.: [hltcoe] HLTCOE evaluation team at TREC 2025: Rag, ragtime, and dragun. In: The Thirty-Fourth Text REtrieval Conference Proceedings (TREC2025) (2025)"},{"key":"37_CR14","doi-asserted-by":"crossref","unstructured":"Dietz, L., et al.: Principles and guidelines for the use of LLM judges. In: Proceedings of the 11th ACM SIGIR \/ The 15th International Conference on Innovative Concepts and Theories in Information Retrieval (2025)","DOI":"10.1145\/3731120.3744588"},{"key":"37_CR15","unstructured":"Ding, Y., et al.: Citations and trust in LLM generated responses (2025). https:\/\/arxiv.org\/abs\/2501.01303"},{"key":"37_CR16","unstructured":"Duh, K., Yang, E., Weller, O., Yates, A., Lawrie, D.: HLTCOE at LiveRAG: GPT-Researcher using ColBERT retrieval. arXiv preprint arXiv:2506.22356 (2025)"},{"key":"37_CR17","unstructured":"Elovic, A.: gpt-researcher (2023). https:\/\/github.com\/assafelovic\/gpt-researcher"},{"key":"37_CR18","unstructured":"Farzi, N., Dietz, L.: Exam++: LLM-based answerability metrics for IR evaluation. In: Proceedings of LLM4Eval: The First Workshop on Large Language Models for Evaluation in Information Retrieval (2024)"},{"key":"37_CR19","doi-asserted-by":"crossref","unstructured":"Farzi, N., Dietz, L.: Pencils down! Automatic rubric-based evaluation of retrieve\/generate systems. In: Proceedings of the 2024 ACM SIGIR International Conference on Theory of Information Retrieval, pp. 175\u2013184 (2024)","DOI":"10.1145\/3664190.3672511"},{"key":"37_CR20","doi-asserted-by":"publisher","unstructured":"Fr\u00f6be, M., et al.: The Information Retrieval Experiment Platform. In: Chen, H.H., Duh, W., Huang, H.H., Kato, M., Mothe, J., Poblete, B. (eds.) 46th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2023), pp. 2826\u20132836, ACM (2023). ISBN 978145039408 https:\/\/doi.org\/10.1145\/3539618.3591888","DOI":"10.1145\/3539618.3591888"},{"key":"37_CR21","doi-asserted-by":"crossref","unstructured":"Izacard, G., Grave, \u00c9.: Leveraging passage retrieval with generative models for open domain question answering. In: Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume, pp. 874\u2013880 (2021)","DOI":"10.18653\/v1\/2021.eacl-main.74"},{"key":"37_CR22","doi-asserted-by":"crossref","unstructured":"Kurland, O., Tennenholtz, M.: Competitive search. In: Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 2838\u20132849, SIGIR \u201922 (2022)","DOI":"10.1145\/3477495.3532771"},{"key":"37_CR23","doi-asserted-by":"crossref","unstructured":"\u0141ajewska, W., Balog, K.: Ginger: Grounded information nugget-based generation of responses. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 2723\u20132727 (2025)","DOI":"10.1145\/3726302.3730166"},{"key":"37_CR24","unstructured":"Lawrie, D., et al.: Overview of the TREC 2024 NeuCLIR track. arXiv preprint arXiv:2509.14355 (2025)"},{"key":"37_CR25","unstructured":"Lewis, P., et al.: Retrieval-augmented generation for knowledge-intensive NLP tasks: Adv. Neural. Inf. Process. Syst. 33, 9459\u20139474 (2020)"},{"key":"37_CR26","unstructured":"Li, D., et al.: From generation to judgment: opportunities and challenges of LLM-as-a-judge (2025). https:\/\/arxiv.org\/abs\/2411.16594"},{"key":"37_CR27","doi-asserted-by":"crossref","unstructured":"Mayfield, J., et al.: On the evaluation of machine-generated reports. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1904\u20131915 (2024)","DOI":"10.1145\/3626772.3657846"},{"key":"37_CR28","unstructured":"Nguyen, T., Lei, Y., Ju, J.H., Yang, E., Yates, A.: Milco: learned sparse retrieval across languages via a multilingual connector. arXiv (2025)"},{"key":"37_CR29","doi-asserted-by":"publisher","unstructured":"Pavlu, V., Rajput, S., Golbus, P.B., Aslam, J.A.: IR system evaluation using nugget-based test collections. In: Proceedings of the fifth ACM International Conference on Web Search and Data Mining, pp. 393\u2013402 (2012). https:\/\/doi.org\/10.1145\/2124295.2124343","DOI":"10.1145\/2124295.2124343"},{"key":"37_CR30","unstructured":"Pradeep, R., Thakur, N., Upadhyay, S., Campos, D., Craswell, N., Lin, J.: Initial nugget evaluation results for the TREC 2024 rag track with the autonuggetizer framework (2024). https:\/\/arxiv.org\/abs\/2411.09607. arXiv preprint"},{"key":"37_CR31","doi-asserted-by":"publisher","unstructured":"Pradeep, R., et al.: The great nugget recall: automating fact extraction and rag evaluation with large language models. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 180\u2013190, SIGIR \u201925, Association for Computing Machinery, New York, NY, USA (2025), ISBN 979840071592 https:\/\/doi.org\/10.1145\/3726302.3730090","DOI":"10.1145\/3726302.3730090"},{"issue":"1","key":"37_CR32","first-page":"10","volume":"1","author":"I Soboroff","year":"2025","unstructured":"Soboroff, I.: Don\u2019t use LLMs to make relevance judgments. Inf. Retrieval Res. J. 1(1), 10\u201354195 (2025)","journal-title":"Inf. Retrieval Res. J."},{"key":"37_CR33","doi-asserted-by":"publisher","unstructured":"Thakur, N., Pradeep, R., Upadhyay, S., Campos, D., Craswell, N., Lin, J.: Support evaluation for the TREC 2024 rag track: Comparing human versus LLM judges. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR \u201925), ACM (2025). https:\/\/doi.org\/10.1145\/3726302.3730165","DOI":"10.1145\/3726302.3730165"},{"key":"37_CR34","doi-asserted-by":"crossref","unstructured":"Thomas, P., Spielman, S., Craswell, N., Mitra, B.: Large language models can accurately predict searcher preferences. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval (2024)","DOI":"10.1145\/3626772.3657707"},{"key":"37_CR35","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Craswell, N., Lin, J.: UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519 (2024)"},{"key":"37_CR36","doi-asserted-by":"publisher","unstructured":"Upadhyay, S., Thakur, N., Pradeep, R., Campos, D., Clarke, C.L.A., Lin, J.: A large-scale study of relevance assessments with large language models using UMBRELA. In: Proceedings of the 15th International Conference on the Theory of Information Retrieval (ICTIR \u201925), ACM (2025). https:\/\/doi.org\/10.1145\/3731120.3744605","DOI":"10.1145\/3731120.3744605"},{"key":"37_CR37","unstructured":"Walden, W., et al.: Auto-ARGUE: LLM-based report generation evaluation. arXiv preprint arXiv:2509.26184 (2025)"},{"key":"37_CR38","doi-asserted-by":"crossref","unstructured":"Yang, E., Lawrie, D., Mayfield, J., Oard, D.W., Miller, S.: Translate-distill: learning cross-language dense retrieval by translation and distillation. In: European Conference on Information Retrieval, pp. 50\u201365, Springer (2024)","DOI":"10.1007\/978-3-031-56060-6_4"},{"key":"37_CR39","unstructured":"Yang, E., Lawrie, D., Weller, O., Mayfield, J.: HLTCOE at TREC 2024 NeuCLIR track (2025). https:\/\/arxiv.org\/abs\/2510.00143"},{"key":"37_CR40","unstructured":"Zhang, Y., et al.: Qwen3 embedding: advancing text embedding and reranking through foundation models. arXiv preprint arXiv:2506.05176 (2025)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-21289-4_37","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T01:06:25Z","timestamp":1774314385000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-21289-4_37"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032212887","9783032212894"],"references-count":40,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-21289-4_37","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"25 March 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"Authors have no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Delft","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 March 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"48","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2026.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}