{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:40:02Z","timestamp":1755866402124,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":44,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100006374","name":"Klaus Tschira Stiftung","doi-asserted-by":"publisher","award":["00.003.2020"],"award-info":[{"award-number":["00.003.2020"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"Deutsche Forschungsgemeinschaft","doi-asserted-by":"publisher","award":["492813820"],"award-info":[{"award-number":["492813820"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,13]]},"DOI":"10.1145\/3726302.3730342","type":"proceedings-article","created":{"date-parts":[[2025,7,14]],"date-time":"2025-07-14T01:38:52Z","timestamp":1752457132000},"page":"3691-3701","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["REANIMATOR: Reanimate Retrieval Test Collections with Extracted and Synthetic Resources"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-7074-9066","authenticated-orcid":false,"given":"Bj\u00f6rn","family":"Engelmann","sequence":"first","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3392-7860","authenticated-orcid":false,"given":"Fabian","family":"Haak","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8817-4632","authenticated-orcid":false,"given":"Philipp","family":"Schaer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8791-6455","authenticated-orcid":false,"given":"Mani","family":"Erfanian Abdoust","sequence":"additional","affiliation":[{"name":"Science Media Center, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4392-9897","authenticated-orcid":false,"given":"Linus","family":"Netze","sequence":"additional","affiliation":[{"name":"Science Media Center, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4668-6030","authenticated-orcid":false,"given":"Meik","family":"Bittkowski","sequence":"additional","affiliation":[{"name":"Science Media Center, Cologne, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-324.core-overview"},{"key":"e_1_3_2_1_2_1","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan Nicholas Joseph Saurav Kadavath Jackson Kernion Tom Conerly Sheer El-Showk Nelson Elhage Zac Hatfield-Dodds Danny Hernandez Tristan Hume Scott Johnston Shauna Kravec Liane Lovitt Neel Nanda Catherine Olsson Dario Amodei Tom Brown Jack Clark Sam McCandlish Chris Olah Ben Mann and Jared Kaplan. 2022. Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback. arXiv:2204.05862 [cs.CL] https:\/\/arxiv.org\/abs\/2204.05862"},{"key":"e_1_3_2_1_3_1","volume-title":"Elo Uncovered: Robustness and Best Practices in Language Model Evaluation. arXiv:2311.17295 [cs.CL]","author":"Boubdir M.","year":"2023","unstructured":"M. Boubdir, E. Kim, B. Ermis, S. Hooker, and M. Fadaee. 2023. Elo Uncovered: Robustness and Best Practices in Language Model Evaluation. arXiv:2311.17295 [cs.CL]"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401036"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657873"},{"key":"e_1_3_2_1_6_1","volume-title":"TabFact: A Largescale Dataset for Table-based Fact Verification. ArXiv (Sept","author":"Chen Wenhu","year":"2019","unstructured":"Wenhu Chen, Hongmin Wang, Jianshu Chen, Yunkai Zhang, Hong Wang, SHIYANG LI, Xiyou Zhou, and William Yang Wang. 2019. TabFact: A Largescale Dataset for Table-based Fact Verification. ArXiv (Sept. 2019). https: \/\/www.semanticscholar.org\/paper\/TabFact%3A-A-Large-scale-Dataset-for- Table-based-Fact-Chen-Wang\/ee4e24bdedd4d2e4be977bd0ca9f68a06ebb4d96? citedSort=relevance&citedPage=2"},{"key":"e_1_3_2_1_7_1","volume-title":"Tianle Li, Dacheng Li, Hao Zhang, Banghua Zhu, Michael Jordan, Joseph E. Gonzalez, and Ion Stoica.","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios Nikolas Angelopoulos, Tianle Li, Dacheng Li, Hao Zhang, Banghua Zhu, Michael Jordan, Joseph E. Gonzalez, and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. arXiv:2403.04132 [cs.AI] https:\/\/arxiv.org\/abs\/2403. 04132"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572114"},{"key":"e_1_3_2_1_9_1","unstructured":"Ganqu Cui Lifan Yuan Ning Ding Guanming Yao Bingxiang He Wei Zhu Yuan Ni Guotong Xie Ruobing Xie Yankai Lin Zhiyuan Liu and Maosong Sun. 2024. UltraFeedback: Boosting Language Models with Scaled AI Feedback. arXiv:2310.01377 [cs.CL] https:\/\/arxiv.org\/abs\/2310.01377"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56060-6_12"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3583780.3615187"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.877"},{"key":"e_1_3_2_1_13_1","volume-title":"RAGAS: Automated Evaluation of Retrieval Augmented Generation. doi:10.48550\/ arXiv.2309.15217 arXiv:2309.15217 [cs].","author":"Es Shahul","year":"2023","unstructured":"Shahul Es, Jithin James, Luis Espinosa-Anke, and Steven Schockaert. 2023. RAGAS: Automated Evaluation of Retrieval Augmented Generation. doi:10.48550\/ arXiv.2309.15217 arXiv:2309.15217 [cs]."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","unstructured":"Kyle Yingkai Gao and Jamie Callan. 2017. Scientific Table Search Using Keyword Queries. doi:10.48550\/arXiv.1707.03423 arXiv:1707.03423.","DOI":"10.48550\/arXiv.1707.03423"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","unstructured":"Yunfan Gao Yun Xiong Xinyu Gao Kangxiang Jia Jinliu Pan Yuxi Bi Yi Dai Jiawei Sun Meng Wang and Haofen Wang. 2024. Retrieval-Augmented Generation for Large Language Models: A Survey. doi:10.48550\/arXiv.2312.10997 arXiv:2312.10997 [cs].","DOI":"10.48550\/arXiv.2312.10997"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.2307\/3608567"},{"key":"e_1_3_2_1_17_1","unstructured":"Jiawei Gu Xuhui Jiang Zhichao Shi Hexiang Tan Xuehao Zhai Chengjin Xu Wei Li Yinghan Shen Shengjie Ma Honghao Liu Saizhuo Wang Kun Zhang Yuanzhuo Wang Wen Gao Lionel Ni and Jian Guo. 2025. A Survey on LLM-asa- Judge. arXiv:2411.15594 [cs.CL] https:\/\/arxiv.org\/abs\/2411.15594"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3630744.3658415"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.43"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672530"},{"key":"e_1_3_2_1_22_1","volume-title":"Overview of the INEX 2013 Social Book Search Track. In Working Notes for CLEF 2013 Conference , Valencia, Spain, September 23-26, 2013 (CEUR Workshop Proceedings","author":"Koolen Marijn","year":"2013","unstructured":"Marijn Koolen, Gabriella Kazai, Michael Preminger, and Antoine Doucet. 2013. Overview of the INEX 2013 Social Book Search Track. In Working Notes for CLEF 2013 Conference , Valencia, Spain, September 23-26, 2013 (CEUR Workshop Proceedings, Vol. 1179), Pamela Forner, Roberto Navigli, Dan Tufis, and Nicola Ferro (Eds.). CEUR-WS.org. https:\/\/ceur-ws.org\/Vol-1179\/CLEF2013wn-INEXKoolenEt2013b. pdf"},{"key":"e_1_3_2_1_23_1","unstructured":"Andreas K\u00f6pf Yannic Kilcher Dimitri von R\u00fctte Sotiris Anagnostidis Zhi- Rui Tam Keith Stevens Abdullah Barhoum Nguyen Minh Duc Oliver Stanley Rich\u00e1rd Nagyfi Shahul ES Sameer Suri David Glushkov Arnav Dantuluri Andrew Maguire Christoph Schuhmann Huu Nguyen and Alexander Mattick. 2023. OpenAssistant Conversations - Democratizing Large Language Model Alignment. arXiv:2304.07327 [cs.CL] https:\/\/arxiv.org\/abs\/2304.07327"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the Third Workshop on Bibliometric-enhanced Information Retrieval co-located with the 38th European Conference on Information Retrieval (ECIR","author":"Larsen Birger","year":"2016","unstructured":"Birger Larsen and Christina Lioma. 2016. On the Need for and Provision for an 'IDEAL' Scholarly Information Retrieval Test Collection. In Proceedings of the Third Workshop on Bibliometric-enhanced Information Retrieval co-located with the 38th European Conference on Information Retrieval (ECIR 2016), Padova, Italy, March 20, 2016 (CEUR Workshop Proceedings, Vol. 1567), Philipp Mayr, Ingo Frommholz, and Guillaume Cabanac (Eds.). CEUR-WS.org, 73-81. https:\/\/ceurws. org\/Vol-1567\/paper8.pdf"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230599.3230612"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463254"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806606"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/S11192-011-0430-X"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009002"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531766"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-65813-1_16"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401120"},{"key":"e_1_3_2_1_35_1","unstructured":"Ian Soboroff. 2024. Don't Use LLMs to Make Relevance Judgments. arXiv:2409.15133 [cs.IR] https:\/\/arxiv.org\/abs\/2409.15133"},{"key":"e_1_3_2_1_36_1","unstructured":"Deep Search Team. 2024. Docling Technical Report. Technical Report. doi:10. 48550\/arXiv.2408.09869 arXiv:2408.09869"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Paul Thomas Seth Spielman Nick Craswell and Bhaskar Mitra. 2024. Large language models can accurately predict searcher preferences. arXiv:2309.10621 [cs.IR] https:\/\/arxiv.org\/abs\/2309.10621","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_38_1","volume-title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv:2406.06519 [cs.IR]","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Nick Craswell, and Jimmy Lin. 2024. UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv:2406.06519 [cs.IR]"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451965"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809453"},{"key":"e_1_3_2_1_41_1","unstructured":"Lucy Lu Wang Kyle Lo Yoganand Chandrasekhar Russell Reas Jiangjiang Yang Darrin Eide K. Funk Rodney Michael Kinney Ziyang Liu W. Merrill P. Mooney D. Murdick Devvret Rishi Jerry Sheehan Zhihong Shen B. Stilson A. Wade K. Wang Christopher Wilhelm Boya Xie D. Raymond Daniel S. Weld Oren Etzioni and Sebastian Kohlmeier. 2020. CORD-19: The Covid-19 Open Research Dataset. ArXiv (2020)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","unstructured":"Xiaohua Wang Zhenghua Wang Xuan Gao Feiran Zhang Yixin Wu Zhibo Xu Tianyuan Shi Zhengyuan Wang Shizheng Li Qi Qian Ruicheng Yin Changze Lv Xiaoqing Zheng and Xuanjing Huang. 2024. Searching for Best Practices in Retrieval-Augmented Generation. doi:10.48550\/arXiv.2407.01219 arXiv:2407.01219 [cs].","DOI":"10.48550\/arXiv.2407.01219"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","unstructured":"Guangzhi Xiong Qiao Jin Zhiyong Lu and Aidong Zhang. 2024. Benchmarking Retrieval-Augmented Generation for Medicine. doi:10.48550\/arXiv.2402.13178 arXiv:2402.13178 [cs].","DOI":"10.48550\/arXiv.2402.13178"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"Hao Yu Aoran Gan Kai Zhang Shiwei Tong Qi Liu and Zhaofeng Liu. 2024. Evaluation of Retrieval-Augmented Generation: A Survey. doi:10.48550\/arXiv. 2405.07437 arXiv:2405.07437 [cs].","DOI":"10.48550\/arXiv"}],"event":{"name":"SIGIR '25: The 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Padua Italy","acronym":"SIGIR '25"},"container-title":["Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3726302.3730342","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:07:30Z","timestamp":1755864450000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3726302.3730342"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":44,"alternative-id":["10.1145\/3726302.3730342","10.1145\/3726302"],"URL":"https:\/\/doi.org\/10.1145\/3726302.3730342","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}