{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T11:48:38Z","timestamp":1774352918619,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":104,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"European Union?s Horizon Europe research and innovation programme (OpenWebSearch.EU)","award":["101070014"],"award-info":[{"award-number":["101070014"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591888","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:23Z","timestamp":1689726143000},"page":"2826-2836","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":30,"title":["The Information Retrieval Experiment Platform"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1003-981X","authenticated-orcid":false,"given":"Maik","family":"Fr\u00f6be","sequence":"first","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1992-8696","authenticated-orcid":false,"given":"Jan Heinrich","family":"Reimer","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8914-2659","authenticated-orcid":false,"given":"Sean","family":"MacAvaney","sequence":"additional","affiliation":[{"name":"University of Glasgow, Glasgow, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6803-1223","authenticated-orcid":false,"given":"Niklas","family":"Deckers","sequence":"additional","affiliation":[{"name":"Leipzig University and ScaDS.AI, Leipzig, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-3036-126X","authenticated-orcid":false,"given":"Simon","family":"Reich","sequence":"additional","affiliation":[{"name":"Leipzig University, Leipzig, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3797-0559","authenticated-orcid":false,"given":"Janek","family":"Bevendorff","sequence":"additional","affiliation":[{"name":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9033-2217","authenticated-orcid":false,"given":"Benno","family":"Stein","sequence":"additional","affiliation":[{"name":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9733-2890","authenticated-orcid":false,"given":"Matthias","family":"Hagen","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2451-0665","authenticated-orcid":false,"given":"Martin","family":"Potthast","sequence":"additional","affiliation":[{"name":"Leipzig University and ScaDS.AI, Leipzig, Germany"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"SIGIR 2015 workshop on repro- ducibility, inexplicability, and generalizability of results (RIGOR). SIGIR 2015. 1147--1148","author":"Arguello J.","unstructured":"J. Arguello, F. Diaz, J. Lin, and A. Trotman. SIGIR 2015 workshop on repro- ducibility, inexplicability, and generalizability of results (RIGOR). SIGIR 2015. 1147--1148."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572153"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914671"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_2_2_6_1","volume-title":"TREC","author":"Benham R.","year":"2018","unstructured":"R. Benham, L. Gallagher, J. M. Mackenzie, B. Liu, X. Lu, F. Scholer, J. Shane Culpepper, and A. Moffat. RMIT at the 2018 TREC CORE Track. TREC 2018."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-76941-7_83"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58219-7_26"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-28241-6_61"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-13643-6_21"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-85251-1_28"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531863"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_58"},{"key":"e_1_3_2_2_14_1","first-page":"32","article-title":"Flexible retrieval with NMSLIB and FlexNeuART","author":"Boytsov L.","year":"2020","unstructured":"L. Boytsov and E. Nyberg. Flexible retrieval with NMSLIB and FlexNeuART. NLP-OSS 2020. 32--43.","journal-title":"NLP-OSS"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401036"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-72240-1_51"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531738"},{"key":"e_1_3_2_2_18_1","volume-title":"STELLA: Towards a framework for the reproducibility of online search experiments. OSIRRC at SIGIR","author":"Breuer T.","year":"2019","unstructured":"T. Breuer, P. Schaer, N. Tavakolpoursaleh, J. Schaible, B. Wolff, and B. M\u00fcller. STELLA: Towards a framework for the reproducibility of online search experiments. OSIRRC at SIGIR 2019. 8--11."},{"key":"e_1_3_2_2_19_1","volume-title":"The TREC 2006 Terabyte track. TREC 2006.","author":"B\u00fcttcher S.","unstructured":"S. B\u00fcttcher, C. L. A. Clarke, and I. Soboroff. The TREC 2006 Terabyte track. TREC 2006."},{"key":"e_1_3_2_2_20_1","volume-title":"Scaling instruction-finetuned language models. arXiv:2210.11416","author":"Chung H. Won","year":"2022","unstructured":"H. Won Chung, L. Hou, S. Longpre, B. Zoph, Y. Tay, W. Fedus, E. Li, X. Wang, M. Dehghani, S. Brahma, A. Webson, S. Shane Gu, Z. Dai, M. Suzgun, X. Chen, A. Chowdhery, S. Narang, G. Mishra, A. Yu, V. Y. Zhao, Y. Huang, A. M. Dai, H. Yu, S. Petrov, E. H. Chi, J. Dean, J. Devlin, A. Roberts, D. Zhou, Q. V. Le, and J. Wei. Scaling instruction-finetuned language models. arXiv:2210.11416 (2022)."},{"key":"e_1_3_2_2_21_1","volume-title":"Overview of the 2019 Open-Source IR Replicability Challenge (OSIRRC","author":"Clancy R.","year":"2019","unstructured":"R. Clancy, N. Ferro, C. Hauff, J. Lin, T. Sakai, and Z. Z. Wu. Overview of the 2019 Open-Source IR Replicability Challenge (OSIRRC 2019). OSIRRC at SIGIR 2019. 1--7."},{"key":"e_1_3_2_2_22_1","volume-title":"Overview of the TREC 2004 Terabyte track. TREC 2004.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, N. Craswell, and I. Soboroff. Overview of the TREC 2004 Terabyte track. TREC 2004."},{"key":"e_1_3_2_2_23_1","volume-title":"Overview of the TREC 2009 Web track. TREC 2009.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, N. Craswell, and I. Soboroff. Overview of the TREC 2009 Web track. TREC 2009."},{"key":"e_1_3_2_2_24_1","volume-title":"Overview of the TREC 2010 Web track. TREC 2010.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, N. Craswell, I. Soboroff, and G. V. Cormack. Overview of the TREC 2010 Web track. TREC 2010."},{"key":"e_1_3_2_2_25_1","volume-title":"Overview of the TREC 2011 Web track. TREC 2011.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, N. Craswell, I. Soboroff, and E. M. Voorhees. Overview of the TREC 2011 Web track. TREC 2011."},{"key":"e_1_3_2_2_26_1","volume-title":"Overview of the TREC 2012 Web track. TREC 2012.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, N. Craswell, and E. M. Voorhees. Overview of the TREC 2012 Web track. TREC 2012."},{"key":"e_1_3_2_2_27_1","volume-title":"The TREC 2005 Terabyte track. TREC 2005.","author":"Clarke C. L. A.","unstructured":"C. L. A. Clarke, F. Scholer, and I. Soboroff. The TREC 2005 Terabyte track. TREC 2005."},{"key":"e_1_3_2_2_28_1","first-page":"192","article-title":"The Cranfield tests on index language devices","volume":"173","author":"Cleverdon C.","year":"1967","unstructured":"C. Cleverdon. The Cranfield tests on index language devices. ASLIB Proceedings, 1967, 173--192.","journal-title":"ASLIB Proceedings"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_3_2_2_30_1","volume-title":"TREC 2013 Web track overview. TREC 2013.","author":"Collins-Thompson K.","unstructured":"K. Collins-Thompson, P. N. Bennett, F. Diaz, C. Clarke, and E. M. Voorhees. TREC 2013 Web track overview. TREC 2013."},{"key":"e_1_3_2_2_31_1","volume-title":"TREC 2014 Web track overview. TREC 2014.","author":"Collins-Thompson K.","unstructured":"K. Collins-Thompson, C. Macdonald, P. N. Bennett, F. Diaz, and E. M. Voorhees. TREC 2014 Web track overview. TREC 2014."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99739-7_33"},{"key":"e_1_3_2_2_33_1","volume-title":"TREC","author":"Craswell N.","year":"2002","unstructured":"N. Craswell and D. Hawking. Overview of the TREC-2002 Web track. TREC 2002."},{"key":"e_1_3_2_2_34_1","volume-title":"Overview of the TREC 2004 Web track. TREC 2004.","author":"Craswell N.","unstructured":"N. Craswell and D. Hawking. Overview of the TREC 2004 Web track. TREC 2004."},{"key":"e_1_3_2_2_35_1","volume-title":"Overview of the TREC 2003 Web track. TREC 2003. 78--92","author":"Craswell N.","unstructured":"N. Craswell, D. Hawking, R. Wilkinson, and M. Wu. Overview of the TREC 2003 Web track. TREC 2003. 78--92."},{"key":"e_1_3_2_2_36_1","volume-title":"Overview of the TREC 2020 Deep Learning track. TREC 2020.","author":"Craswell N.","unstructured":"N. Craswell, B. Mitra, E. Yilmaz, and D. Campos. Overview of the TREC 2020 Deep Learning track. TREC 2020."},{"key":"e_1_3_2_2_37_1","volume-title":"Overview of the TREC 2019 Deep Learning track. TREC 2019.","author":"Craswell N.","unstructured":"N. Craswell, B. Mitra, E. Yilmaz, D. Campos, and E. M. Voorhees. Overview of the TREC 2019 Deep Learning track. TREC 2019."},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2964797.2964808"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28577-7_24"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-98932-7_23"},{"key":"e_1_3_2_2_41_1","volume-title":"SPLADE v2: Sparse lexical and expansion model for information retrieval. arXiv:2109.10086","author":"Formal T.","year":"2021","unstructured":"T. Formal, C. Lassance, B. Piwowarski, and S. Clinchant. SPLADE v2: Sparse lexical and expansion model for information retrieval. arXiv:2109.10086 (2021)."},{"key":"e_1_3_2_2_42_1","first-page":"2278","article-title":"SemEval-2023 task 5: Clickbait spoiling","author":"Fr\u00f6be M.","year":"2023","unstructured":"M. Fr\u00f6be, T. Gollub, M. Hagen, and M. Potthast. SemEval-2023 task 5: Clickbait spoiling. SemEval-2023. 2278--2289.","journal-title":"SemEval-"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-28241-6_20"},{"key":"e_1_3_2_2_44_1","first-page":"3","article-title":"Some common mistakes in IR evaluation, and how they can be avoided","volume":"51","author":"Fuhr N.","year":"2017","unstructured":"N. Fuhr. Some common mistakes in IR evaluation, and how they can be avoided. SIGIR Forum 51, 3 (2017), 32--41.","journal-title":"SIGIR Forum"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3483382.3483385"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3402426"},{"key":"e_1_3_2_2_47_1","volume-title":"Precise zero-shot dense retrieval without relevance labels. arXiv:2212.10496","author":"Gao L.","year":"2022","unstructured":"L. Gao, X. Ma, J. Lin, and J. Callan. Precise zero-shot dense retrieval without relevance labels. arXiv:2212.10496 (2022)."},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"crossref","unstructured":"T. Gollub B. Stein S. Burrows and D. Hoppe. TIRA: Configuring executing and disseminating information retrieval experiments. TIR 2012 at DEXA. 151--155.","DOI":"10.1109\/DEXA.2012.55"},{"key":"e_1_3_2_2_49_1","volume-title":"CLEF","author":"Hagen M.","year":"2017","unstructured":"M. Hagen, M. Potthast, and B. Stein. Overview of the author obfuscation task at PAN 2017: Safety evaluation revisited. CLEF 2017. 1613--0073."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-45442-5_21"},{"key":"e_1_3_2_2_51_1","volume-title":"TREC 2004 Genomics track overview. TREC 2004.","author":"Hersh W. R.","unstructured":"W. R. Hersh, R. Teja Bhupatiraju, L. Ross, A. M. Cohen, D. Kraemer, and P. Johnson. TREC 2004 Genomics track overview. TREC 2004."},{"key":"e_1_3_2_2_52_1","volume-title":"TREC 2005 Genomics track overview. TREC 2005.","author":"Hersh W. R.","unstructured":"W. R. Hersh, A. M. Cohen, J. Yang, R. Teja Bhupatiraju, P. M. Roberts, and M. A. Hearst. TREC 2005 Genomics track overview. TREC 2005."},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462891"},{"key":"e_1_3_2_2_54_1","first-page":"2","article-title":"Benchmarking news recommendations: The CLEF NewsREEL use case","volume":"49","author":"Hopfgartner F.","year":"2015","unstructured":"F. Hopfgartner, T. Brodt, J. Seiler, B. Kille, A. Lommatzsch, M. A. Larson, R. Turrin, and A. Ser\u00e9ny. Benchmarking news recommendations: The CLEF NewsREEL use case. SIGIR Forum 49, 2 (2015), 129--136.","journal-title":"SIGIR Forum"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3239570"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3239575"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462784"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401075"},{"key":"e_1_3_2_2_60_1","first-page":"2290","article-title":"SemEval-2023 task 4: ValueEval: Identification of human values behind arguments","author":"Kiesel J.","year":"2023","unstructured":"J. Kiesel, M. Alshomary, N. Mirzakhmedova, M. Heinrich, N. Handke, H. Wachsmuth, and B. Stein. SemEval-2023 task 4: ValueEval: Identification of human values behind arguments. SemEval-2023. 2290--2306.","journal-title":"SemEval-"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2021.100322"},{"key":"e_1_3_2_2_62_1","first-page":"2","article-title":"The neural hype and comparisons against weak baselines","volume":"52","author":"Lin J.","year":"2018","unstructured":"J. Lin. The neural hype and comparisons against weak baselines. SIGIR Forum 52, 2 (2018), 40--51.","journal-title":"SIGIR Forum"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531725"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463238"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-45442-5_6"},{"key":"e_1_3_2_2_66_1","volume-title":"ECIR","author":"MacAvaney S.","year":"2022","unstructured":"S. MacAvaney, C. Macdonald, and I. Ounis. Streamlining evaluation with ir_measures. ECIR 2022. 305--310."},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371864"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463254"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482013"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409829"},{"key":"e_1_3_2_2_71_1","volume-title":"PISA: Performant indexes and search for academia. OSIRRC at SIGIR","author":"Mallia A.","year":"2019","unstructured":"A. Mallia, M. Siedlaczek, J. M. Mackenzie, and T. Suel. PISA: Performant indexes and search for academia. OSIRRC at SIGIR 2019. 50--56."},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2022.3211668"},{"key":"e_1_3_2_2_73_1","volume-title":"MS MARCO: A human cenerated machine reading comprehension dataset. CoCo at NIPS","author":"Nguyen T.","year":"2016","unstructured":"T. Nguyen, M. Rosenberg, X. Song, J. Gao, S. Tiwary, R. Majumder, and L. Deng. MS MARCO: A human cenerated machine reading comprehension dataset. CoCo at NIPS 2016."},{"key":"e_1_3_2_2_74_1","volume-title":"Multi-stage document ranking with BERT. arXiv:1910.14424","author":"Nogueira R. Frassetto","year":"2019","unstructured":"R. Frassetto Nogueira, W. Yang, K. Cho, and J. Lin. Multi-stage document ranking with BERT. arXiv:1910.14424 (2019)."},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-31865-1_37"},{"key":"e_1_3_2_2_76_1","volume-title":"SIGIR","author":"Piwowarski B.","year":"2020","unstructured":"B. Piwowarski. Experimaestro and Datamaestro: Experiment and dataset managers (for IR). SIGIR 2020. 2173--2176."},{"key":"e_1_3_2_2_77_1","volume-title":"TIRA integrated research architecture. Information Retrieval Evaluation in a Changing World","author":"Potthast M.","year":"2019","unstructured":"M. Potthast, T. Gollub, M. Wiegmann, and B. Stein. TIRA integrated research architecture. Information Retrieval Evaluation in a Changing World, 2019. 123--160."},{"key":"e_1_3_2_2_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462798"},{"key":"e_1_3_2_2_79_1","volume-title":"The Expando-Mono-Duo design pattern for text ranking with pretrained sequence-to-sequence models. arXiv:2101.05667","author":"Pradeep R.","year":"2021","unstructured":"R. Pradeep, R. Nogueira, and J. Lin. The Expando-Mono-Duo design pattern for text ranking with pretrained sequence-to-sequence models. arXiv:2101.05667 (2021)."},{"key":"e_1_3_2_2_80_1","first-page":"3980","article-title":"Sentence-BERT: Sentence embeddings using Siamese BERT-networks","author":"Reimers N.","year":"2019","unstructured":"N. Reimers and I. Gurevych. Sentence-BERT: Sentence embeddings using Siamese BERT-networks. EMNLP-IJCNLP 2019. 3980--3990.","journal-title":"EMNLP-IJCNLP"},{"key":"e_1_3_2_2_81_1","volume-title":"Overview of the TREC 2018 Precision Medicine track. TREC 2018.","author":"Roberts K.","unstructured":"K. Roberts, D. Demner-Fushman, E. M. Voorhees, W. R. Hersh, S. Bedrick, and A. J. Lazar. Overview of the TREC 2018 Precision Medicine track. TREC 2018."},{"key":"e_1_3_2_2_82_1","volume-title":"Overview of the TREC 2017 Precision Medicine track. TREC 2017.","author":"Roberts K.","unstructured":"K. Roberts, D. Demner-Fushman, E. M. Voorhees, W. R. Hersh, S. Bedrick, A. J. Lazar, and S. Pant. Overview of the TREC 2017 Precision Medicine track. TREC 2017."},{"key":"e_1_3_2_2_83_1","volume-title":"12:1--12:8","author":"Sakai T.","year":"2020","unstructured":"T. Sakai. On Fuhr's guideline for IR evaluation. SIGIR Forum 54, 1 (2020), 12:1--12:8."},{"key":"e_1_3_2_2_84_1","volume-title":"NTCIR","author":"Sakai T.","year":"2019","unstructured":"T. Sakai, N. Ferro, I. Soboroff, Z. Zeng, P. Xiao, and M. Maistro. Overview of the NTCIR-14 CENTRE task. NTCIR 2019."},{"key":"e_1_3_2_2_85_1","volume-title":"NTCIR","author":"Sakai T.","year":"2020","unstructured":"T. Sakai, S. Tao, Z. Zeng, Y. Zheng, J. Mao, Z. Chu, Y. Liu, M. Maistro, Z. Dou, N. Ferro, et al. Overview of the NTCIR-15 We Want Web with CENTRE (WWW-3) task. NTCIR 2020."},{"key":"e_1_3_2_2_86_1","volume-title":"BEIR: A het- erogeneous benchmark for zero-shot evaluation of information retrieval models. NeurIPS Datasets and Benchmarks","author":"Thakur N.","year":"2021","unstructured":"N. Thakur, N. Reimers, A. R\u00fcckl\u00e9, A. Srivastava, and I. Gurevych. BEIR: A het- erogeneous benchmark for zero-shot evaluation of information retrieval models. NeurIPS Datasets and Benchmarks 2021."},{"key":"e_1_3_2_2_87_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12859-015-0564-6"},{"key":"e_1_3_2_2_88_1","first-page":"2","article-title":"OpenML: Networked science in machine learning","volume":"15","author":"Vanschoren J.","year":"2013","unstructured":"J. Vanschoren, J. N. van Rijn, B. Bischl, and L. Torgo. OpenML: Networked science in machine learning. SIGKDD Explor. 15, 2 (2013), 49--60.","journal-title":"SIGKDD Explor."},{"key":"e_1_3_2_2_89_1","volume-title":"Overview of the TREC 2004 Robust Retrieval track. TREC 2004.","author":"Voorhees E.","unstructured":"E. Voorhees. Overview of the TREC 2004 Robust Retrieval track. TREC 2004."},{"key":"e_1_3_2_2_90_1","volume-title":"Retrieval test collections document set","author":"Voorhees E. M.","year":"1996","unstructured":"E. M. Voorhees. NIST TREC Disks 4 and 5: Retrieval test collections document set. 1996."},{"key":"e_1_3_2_2_91_1","volume-title":"CLEF","author":"Voorhees E. M.","year":"2001","unstructured":"E. M. Voorhees. The philosophy of information retrieval evaluation. CLEF 2001. 355--370."},{"key":"e_1_3_2_2_92_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-22948-1_2"},{"key":"e_1_3_2_2_93_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3451964.3451965","article-title":"Constructing a pandemic information retrieval test collection","volume":"54","author":"Voorhees E. M.","year":"2020","unstructured":"E. M. Voorhees, T. Alam, S. Bedrick, D. Demner-Fushman, W. R. Hersh, K. Lo, K. Roberts, I. Soboroff, and L. Lu Wang. TREC-COVID: Constructing a pandemic information retrieval test collection. SIGIR Forum 54, 1 (2020), 1:1--1:12.","journal-title":"SIGIR Forum"},{"key":"e_1_3_2_2_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531728"},{"key":"e_1_3_2_2_95_1","volume-title":"TREC","author":"Voorhees E. M.","year":"1998","unstructured":"E. M. Voorhees and D. Harman. Overview of the seventh text retrieval conference (TREC-7). TREC 1998."},{"key":"e_1_3_2_2_96_1","volume-title":"TREC","author":"Voorhees E. M.","year":"1999","unstructured":"E. M. Voorhees and D. Harman. Overview of the eighth text retrieval conference (TREC-8). TREC 1999."},{"key":"e_1_3_2_2_97_1","volume-title":"Can old TREC collections reliably evaluate modern neural retrieval models? arXiv:2201.11086","author":"Voorhees E. M.","year":"2022","unstructured":"E. M. Voorhees, I. Soboroff, and J. Lin. Can old TREC collections reliably evaluate modern neural retrieval models? arXiv:2201.11086 (2022)."},{"key":"e_1_3_2_2_98_1","volume-title":"CORD-19: The Covid-19 open research dataset. arXiv:2004.10706","author":"Wang L. Lu","year":"2020","unstructured":"L. Lu Wang, K. Lo, Y. Chandrasekhar, R. Reas, J. Yang, D. Eide, K. Funk, R. Kinney, Z. Liu, W. Merrill, P. Mooney, D. A. Murdick, D. Rishi, J. Sheehan, Z. Shen, B. Stilson, A. D. Wade, K. Wang, C. Wilhelm, B. Xie, D. Raymond, D. S. Weld, O. Etzioni, and S. Kohlmeier. CORD-19: The Covid-19 open research dataset. arXiv:2004.10706 (2020)."},{"key":"e_1_3_2_2_99_1","volume-title":"ICLR","author":"Xiong L.","year":"2021","unstructured":"L. Xiong, C. Xiong, Y. Li, K. Tang, J. Liu, P. N. Bennett, J. Ahmed, and A. Over-wijk. Approximate nearest neighbor negative contrastive learning for dense text retrieval. ICLR 2021."},{"key":"e_1_3_2_2_100_1","volume-title":"EvalAI: Towards better evaluation systems for AI agents. arXiv:1902.03570","author":"Yadav D.","year":"2019","unstructured":"D. Yadav, R. Jain, H. Agrawal, P. Chattopadhyay, T. Singh, A. Jain, S. Singh, S. Lee, and D. Batra. EvalAI: Towards better evaluation systems for AI agents. arXiv:1902.03570 (2019)."},{"key":"e_1_3_2_2_101_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080721"},{"key":"e_1_3_2_2_102_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371868"},{"key":"e_1_3_2_2_103_1","volume-title":"Making a MIRACL: Multilingual informtion retrieval across a continuum of languages. arXiv:2210.09984","author":"Zhang X.","year":"2022","unstructured":"X. Zhang, N. Thakur, O. Ogundepo, E. Kamalloo, D. Alfonso-Hermelo, X. Li, Q. Liu, M. Rezagholizadeh, and J. Lin. Making a MIRACL: Multilingual informtion retrieval across a continuum of languages. arXiv:2210.09984 (2022)."},{"key":"e_1_3_2_2_104_1","doi-asserted-by":"publisher","DOI":"10.1145\/3582524.3582540"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591888","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591888","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:07Z","timestamp":1750178287000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591888"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":104,"alternative-id":["10.1145\/3539618.3591888","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591888","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}