{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T18:31:03Z","timestamp":1775068263402,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":91,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3463254","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T02:41:52Z","timestamp":1626057712000},"page":"2429-2436","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":84,"title":["Simplified Data Wrangling with ir_datasets"],"prefix":"10.1145","author":[{"given":"Sean","family":"MacAvaney","sequence":"first","affiliation":[{"name":"University of Glasgow & Georgetown University, Glasgow, United Kingdom"}]},{"given":"Andrew","family":"Yates","sequence":"additional","affiliation":[{"name":"Max Planck Institute for Informatics, Saarbr\u00fccken, Germany"}]},{"given":"Sergey","family":"Feldman","sequence":"additional","affiliation":[{"name":"Allen Institute for AI, Seattle, WA, USA"}]},{"given":"Doug","family":"Downey","sequence":"additional","affiliation":[{"name":"Allen Institute for AI, Seattle, WA, USA"}]},{"given":"Arman","family":"Cohan","sequence":"additional","affiliation":[{"name":"Allen Institute for AI, Seattle, WA, USA"}]},{"given":"Nazli","family":"Goharian","sequence":"additional","affiliation":[{"name":"Georgetown University, Washington, DC, USA"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Overview of the TREC 2019 Decision Track. In TREC.","author":"Abualsaud Mustafa","year":"2019","unstructured":"Mustafa Abualsaud, Christina Lioma, Maria Maistro, Mark D. Smucker, and Guido Zuccon. 2019. Overview of the TREC 2019 Decision Track. In TREC."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"James Allan Javed A. Aslam Ben Carterette Virgil Pavlu and Evangelos Kanoulas. 2008. Million Query Track 2008 Overview. In TREC.","DOI":"10.21236\/ADA477388"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"James Allan Ben Carterette Javed A. Aslam Virgil Pavlu Blagovest Dachev and Evangelos Kanoulas. 2007. Million Query Track 2007 Overview. In TREC.","DOI":"10.21236\/ADA477388"},{"key":"e_1_3_2_1_4_1","volume-title":"TREC 2017 Common Core Track Overview. In TREC.","author":"Allan James","year":"2017","unstructured":"James Allan, Donna Harman, Evangelos Kanoulas, Dan Li, Christophe Van Gysel, and Ellen Vorhees. 2017. TREC 2017 Common Core Track Overview. In TREC."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Alexander Bondarenko Maik Fr\u00f6be Meriem Beloucif Lukas Gienapp Yamen Ajjour Alexander Panchenko Christian Biemann Benno Stein Henning Wachsmuth Martin Potthast and Matthias Hagen. 2020. Overview of Touch\u00e9 2020: Argument Retrieval. In CLEF.","DOI":"10.1007\/978-3-030-58219-7_26"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_58"},{"key":"e_1_3_2_1_7_1","volume-title":"The TREC 2006 Terabyte Track. In TREC.","author":"B\u00fcttcher Stefan","year":"2006","unstructured":"Stefan B\u00fcttcher, Charles L. A. Clarke, and Ian Soboroff. 2006. The TREC 2006 Terabyte Track. In TREC."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Ben Carterette Virgil Pavlu Hui Fang and Evangelos Kanoulas. 2009. Million Query Track 2009 Overview. In TREC.","DOI":"10.6028\/NIST.SP.500-278.million-query-overview"},{"key":"e_1_3_2_1_9_1","volume-title":"The TREC 2005 Terabyte Track. In TREC.","author":"Clark Charles L. A.","year":"2005","unstructured":"Charles L. A. Clark, Falk Scholer, and Ian Soboroff. 2005. The TREC 2005 Terabyte Track. In TREC."},{"key":"e_1_3_2_1_10_1","volume-title":"Overview of the TREC 2004 Terabyte Track. In TREC.","author":"Clarke Charles","year":"2004","unstructured":"Charles Clarke, Nick Craswell, and Ian Soboroff. 2004. Overview of the TREC 2004 Terabyte Track. In TREC."},{"key":"e_1_3_2_1_11_1","volume-title":"Overview of the TREC 2009 Web Track. In TREC.","author":"Clarke Charles L. A.","year":"2009","unstructured":"Charles L. A. Clarke, Nick Craswell, and Ian Soboroff. 2009. Overview of the TREC 2009 Web Track. In TREC."},{"key":"e_1_3_2_1_12_1","volume-title":"Overview of the TREC 2010 Web Track. In TREC.","author":"Clarke Charles L. A.","unstructured":"Charles L. A. Clarke, Nick Craswell, Ian Soboroff, and Gordon V. Cormack. 2010. Overview of the TREC 2010 Web Track. In TREC."},{"key":"e_1_3_2_1_13_1","volume-title":"Overview of the TREC 2011 Web Track. In TREC.","author":"Clarke Charles L. A.","unstructured":"Charles L. A. Clarke, Nick Craswell, Ian Soboroff, and Ellen M. Voorhees. 2011. Overview of the TREC 2011 Web Track. In TREC."},{"key":"e_1_3_2_1_14_1","volume-title":"Overview of the TREC 2012 Web Track. In TREC.","author":"Clarke Charles L. A.","unstructured":"Charles L. A. Clarke, Nick Craswell, and Ellen M. Voorhees. 2012. Overview of the TREC 2012 Web Track. In TREC."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_16_1","volume-title":"TREC 2013 Web Track Overview. In TREC.","author":"Collins-Thompson Kevyn","unstructured":"Kevyn Collins-Thompson, Paul Bennett, Fernando Diaz, Charles L. A. Clarke, and Ellen M. Voorhees. 2013. TREC 2013 Web Track Overview. In TREC."},{"key":"e_1_3_2_1_17_1","volume-title":"TREC 2014 Web Track Overview. In TREC.","author":"Collins-Thompson Kevyn","unstructured":"Kevyn Collins-Thompson, Craig Macdonald, Paul Bennett, Fernando Diaz, and Ellen M. Voorhees. 2014. TREC 2014 Web Track Overview. In TREC."},{"key":"e_1_3_2_1_18_1","volume-title":"2020 a. ORCAS: 18 Million Clicked Query-Document Pairs for Analyzing Search. arXiv preprint arXiv:2006.05324","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Daniel Campos, Bhaskar Mitra, Emine Yilmaz, and Bodo Billerbeck. 2020 a. ORCAS: 18 Million Clicked Query-Document Pairs for Analyzing Search. arXiv preprint arXiv:2006.05324 (2020)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Nick Craswell and David Hawking. 2002. Overview of the TREC-2002 Web Track. In TREC.","DOI":"10.6028\/NIST.SP.500-251.web-overview"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Nick Craswell and David Hawking. 2004. Overview of the TREC-2004 Web Track. In TREC.","DOI":"10.6028\/NIST.SP.500-261.web-overview"},{"key":"e_1_3_2_1_21_1","volume-title":"Overview of the TREC 2003 Web Track. In TREC.","author":"Craswell Nick","year":"2003","unstructured":"Nick Craswell, David Hawking, Ross Wilkinson, and Mingfang Wu. 2003. Overview of the TREC 2003 Web Track. In TREC."},{"key":"e_1_3_2_1_22_1","volume-title":"Overview of the TREC 2020 deep learning track. In TREC.","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, and Daniel Campos. 2020 b. Overview of the TREC 2020 deep learning track. In TREC."},{"key":"e_1_3_2_1_23_1","volume-title":"Overview of the TREC 2019 deep learning track. In TREC 2019.","author":"Craswell Nick","year":"2019","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen Voorhees. 2019. Overview of the TREC 2019 deep learning track. In TREC 2019."},{"key":"e_1_3_2_1_24_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In NAACL-HLT.","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In NAACL-HLT."},{"key":"e_1_3_2_1_25_1","volume-title":"TREC CAR: A Data Set for Complex Answer Retrieval.","author":"Dietz Laura","year":"2017","unstructured":"Laura Dietz and Ben Gamari. 2017. TREC CAR: A Data Set for Complex Answer Retrieval. (2017). http:\/\/trec-car.cs.unh.edu Version 1.5."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Laura Dietz Manisha Verma Filip Radlinski and Nick Craswell. 2017. TREC Complex Answer Retrieval Overview.. In TREC.","DOI":"10.6028\/NIST.SP.500-324.car-overview"},{"key":"e_1_3_2_1_27_1","volume-title":"CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims. ArXiv","author":"Diggelmann T.","year":"2020","unstructured":"T. Diggelmann, Jordan L. Boyd-Graber, Jannis Bulian, Massimiliano Ciaramita, and Markus Leippold. 2020. CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims. ArXiv, Vol. abs\/2012.00614 (2020)."},{"key":"e_1_3_2_1_28_1","volume-title":"2020 a","author":"Frej Jibril","unstructured":"Jibril Frej, Didier Schwab, and Jean-Pierre Chevallet. 2020 a. MLWIKIR: A Python Toolkit for Building Large-scale Wikipedia-based Information Retrieval Datasets in Chinese, English, French, Italian, Japanese, Spanish and More. In CIRCLE."},{"key":"e_1_3_2_1_29_1","unstructured":"Jibril Frej Didier Schwab and Jean-Pierre Chevallet. 2020 b. WIKIR: A Python toolkit for building a large-scale Wikipedia-based English Information Retrieval Dataset. In LREC."},{"key":"e_1_3_2_1_30_1","unstructured":"Fredric Gey and Douglas Oard. 2001. The TREC-2001 Cross-Language Information Retrieval Track: Searching Arabic using English French or Arabic Queries. In TREC."},{"key":"e_1_3_2_1_31_1","unstructured":"Fredric Gey and Douglas Oard. 2002. The TREC-2002 Arabic\/English CLIR Track. In TREC."},{"key":"e_1_3_2_1_32_1","volume-title":"Arabic Newswire Part 1 LDC2001T55. https:\/\/catalog.ldc.upenn.edu\/LDC2001T55","author":"Graff David","unstructured":"David Graff and Kevin Walker. 2001. Arabic Newswire Part 1 LDC2001T55. https:\/\/catalog.ldc.upenn.edu\/LDC2001T55"},{"key":"e_1_3_2_1_33_1","volume-title":"Overview of the Third Text REtrieval Conference (TREC-3). In TREC.","author":"Harman Donna","year":"1994","unstructured":"Donna Harman. 1994. Overview of the Third Text REtrieval Conference (TREC-3). In TREC."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-225"},{"key":"e_1_3_2_1_35_1","volume-title":"ANTIQUE: A Non-Factoid Question Answering Benchmark. In ECIR.","author":"Hashemi Helia","year":"2020","unstructured":"Helia Hashemi, Mohammad Aliannejadi, Hamed Zamani, and Bruce Croft. 2020. ANTIQUE: A Non-Factoid Question Answering Benchmark. In ECIR."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080751"},{"key":"e_1_3_2_1_37_1","volume-title":"TREC 2007 Genomics Track Overview. In TREC.","author":"Hersh William","year":"2007","unstructured":"William Hersh, Aaron Cohen, Lynn Ruslen, and Phoebe Roberts. 2007 a. TREC 2007 Genomics Track Overview. In TREC."},{"key":"e_1_3_2_1_38_1","volume-title":"TREC 2005 Genomics Track Overview. In TREC.","author":"Hersh William","year":"2007","unstructured":"William Hersh, Aaron Cohen, Jianji Yang, Ravi Teja Bhupatiraju, Phoebe Roberts, and Marti Hearst. 2007 b. TREC 2005 Genomics Track Overview. In TREC."},{"key":"e_1_3_2_1_39_1","volume-title":"TREC 2006 Genomics Track Overview. In TREC.","author":"Hersh William","year":"2006","unstructured":"William Hersh, Aaron M. Cohen, Phoebe Roberts, and Hari Krishna Rekapalli. 2006. TREC 2006 Genomics Track Overview. In TREC."},{"key":"e_1_3_2_1_40_1","volume-title":"TREC 2004 Genomics Track Overview. In TREC.","author":"Hersh William R.","unstructured":"William R. Hersh, Ravi Teja Bhuptiraju, Laura Ross, Phoebe Johnson, Aaron M. Cohen, and Dale F. Kraemer. 2004. TREC 2004 Genomics Track Overview. In TREC."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2838931.2838934"},{"key":"e_1_3_2_1_42_1","volume-title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search. ArXiv","author":"Husain Hamel","year":"2019","unstructured":"Hamel Husain, Ho-Hsiang Wu, Tiferet Gazit, Miltiadis Allamanis, and Marc Brockschmidt. 2019. CodeSearchNet Challenge: Evaluating the State of Semantic Code Search. ArXiv (2019)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462784"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"crossref","unstructured":"Mandar Joshi Eunsol Choi Daniel S. Weld and Luke Zettlemoyer. 2017. TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension. In ACL.","DOI":"10.18653\/v1\/P17-1147"},{"key":"e_1_3_2_1_45_1","volume-title":"Dense Passage Retrieval for Open-Domain Question Answering. arxiv","author":"Karpukhin Vladimir","year":"2004","unstructured":"Vladimir Karpukhin, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen tau Yih. 2020. Dense Passage Retrieval for Open-Domain Question Answering. arxiv: 2004.04906 [cs.CL]"},{"key":"e_1_3_2_1_46_1","volume-title":"ColBERT: Efficient and Effective Passage Search via Contextualized Late Interaction over BERT. SIGIR","author":"Khattab Omar","year":"2020","unstructured":"Omar Khattab and Matei Zaharia. 2020. ColBERT: Efficient and Effective Passage Search via Contextualized Late Interaction over BERT. SIGIR (2020)."},{"key":"e_1_3_2_1_47_1","volume-title":"Natural Questions: a Benchmark for Question Answering Research. TACL","author":"Kwiatkowski Tom","year":"2019","unstructured":"Tom Kwiatkowski, Jennimaria Palomaki, Olivia Redfield, Michael Collins, Ankur Parikh, Chris Alberti, Danielle Epstein, Illia Polosukhin, Matthew Kelcey, Jacob Devlin, Kenton Lee, Kristina N. Toutanova, Llion Jones, Ming-Wei Chang, Andrew Dai, Jakob Uszkoreit, Quoc Le, and Slav Petrov. 2019. Natural Questions: a Benchmark for Question Answering Research. TACL (2019)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Jimmy Lin and Miles Efron. 2013. Overview of the TREC-2013 Microblog Track. In TREC.","DOI":"10.6028\/NIST.SP.500-302.microblog-overview"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"Jimmy Lin Miles Efron Yulu Wang and Garrick Sherman. 2014. Overview of the TREC-2014 Microblog Track. In TREC.","DOI":"10.6028\/NIST.SP.500-308.microblog-overview"},{"key":"e_1_3_2_1_50_1","volume-title":"Pyserini: An Easy-to-Use Python Toolkit to Support Replicable IR Research with Sparse and Dense Representations. ArXiv","author":"Lin Jimmy","year":"2021","unstructured":"Jimmy Lin, Xueguang Ma, Sheng-Chieh Lin, Jheng-Hong Yang, Ronak Pradeep, and Rodrigo Nogueira. 2021. Pyserini: An Easy-to-Use Python Toolkit to Support Replicable IR Research with Sparse and Dense Representations. ArXiv, Vol. abs\/2102.10073 (2021)."},{"key":"e_1_3_2_1_51_1","unstructured":"Cheng Luo Tetsuya Sakai Yiqun Liu Zhicheng Dou Chenyan Xiong and Jingfang Xu. 2017. Overview of the NTCIR-13 We Want Web Task. In NTCIR."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371864"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401093"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Sean MacAvaney Andrew Yates Kai Hui and Ophir Frieder. 2019. Content-Based Weak Supervision for Ad-Hoc Re-Ranking. In SIGIR.","DOI":"10.1145\/3331184.3331316"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409829"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463262"},{"key":"e_1_3_2_1_57_1","volume-title":"WWW'18 Open Challenge: Financial Opinion Mining and Question Answering. Companion Proceedings of the The Web Conference 2018","author":"Maia Macedo","year":"2018","unstructured":"Macedo Maia, S. Handschuh, A. Freitas, Brian Davis, R. McDermott, M. Zarrouk, and A. Balahur. 2018. WWW'18 Open Challenge: Financial Opinion Mining and Question Answering. Companion Proceedings of the The Web Conference 2018 (2018)."},{"key":"e_1_3_2_1_58_1","volume-title":"Proceedings of the Open-Source IR Replicability Challenge co-located with 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, OSIRRC@SIGIR 2019","author":"Mallia Antonio","year":"2019","unstructured":"Antonio Mallia, Michal Siedlaczek, Joel Mackenzie, and Torsten Suel. 2019. PISA: Performant Indexes and Search for Academia. In Proceedings of the Open-Source IR Replicability Challenge co-located with 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, OSIRRC@SIGIR 2019, Paris, France, July 25, 2019. 50--56. http:\/\/ceur-ws.org\/Vol-2409\/docker08.pdf"},{"key":"e_1_3_2_1_59_1","unstructured":"Jiaxin Mao Tetsuya Sakai Cheng Luo Peng Xiao Yiqun Liu and Zhicheng Dou. 2018. Overview of the NTCIR-14 We Want Web Task. In NTCIR."},{"key":"e_1_3_2_1_60_1","volume-title":"Terrier information retrieval platform","author":"Ounis Iadh","unstructured":"Iadh Ounis, Gianni Amati, Vassilis Plachouras, Ben He, Craig Macdonald, and Douglas Johnson. 2005. Terrier information retrieval platform. In ECIR. Springer, 517--519."},{"key":"e_1_3_2_1_61_1","volume-title":"CLEF 2017 Task Overview: The IR Task at the eHealth Evaluation Lab - Evaluating Retrieval Methods for Consumer Health Search. In CLEF.","author":"Palotti Joao","year":"2017","unstructured":"Joao Palotti, Guido Zuccon, Jimmy, Pavel Pecina, Mihai Lupu, Lorraine Goeuriot, Liadh Kelly, and Allan Hanbury. 2017. CLEF 2017 Task Overview: The IR Task at the eHealth Evaluation Lab - Evaluating Retrieval Methods for Consumer Health Search. In CLEF."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","unstructured":"The pandas development team. 2020. pandas-dev\/pandas: Pandas. https:\/\/doi.org\/10.5281\/zenodo.3509134","DOI":"10.5281\/zenodo.3509134"},{"key":"e_1_3_2_1_63_1","volume-title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. In InCoCo@NIPS.","author":"Li Deng Jianfeng Gao Nick Craswell","year":"2016","unstructured":"Nick Craswell Li Deng Jianfeng Gao Xiaodong Liu Rangan Majumder Andrew McNamara Bhaskar Mitra Tri Nguyen Mir Rosenberg Xia Song Alina Stoica Saurabh Tiwary Tong Wang Payal Bajaj, Daniel Campos. 2016. MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. In InCoCo@NIPS."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"crossref","unstructured":"Navid Rekabsaz Oleg Lesota Markus Schedl Jon Brassey and Carsten Eickhoff. 2021. TripClick: The Log Files of a Large Health Web Search Engine. In SIGIR.","DOI":"10.1145\/3404835.3463242"},{"key":"e_1_3_2_1_65_1","volume-title":"Overview of the TREC 2018 Precision Medicine Track. In TREC.","author":"Roberts Kirk","unstructured":"Kirk Roberts, Dina Demner-Fushman, Ellen Voorhees, William R. Hersh, Steven Bedrick, and Alexander J. Lazar. 2018. Overview of the TREC 2018 Precision Medicine Track. In TREC."},{"key":"e_1_3_2_1_66_1","volume-title":"Overview of the TREC 2017 Precision Medicine Track. In TREC.","author":"Roberts Kirk","year":"2017","unstructured":"Kirk Roberts, Dina Demner-Fushman, Ellen Voorhees, William R. Hersh, Steven Bedrick, Alexander J. Lazar, and Shubham Pant. 2017. Overview of the TREC 2017 Precision Medicine Track. In TREC."},{"key":"e_1_3_2_1_67_1","volume-title":"Overview of the TREC 2019 Precision Medicine Track. In TREC.","author":"Roberts Kirk","year":"2019","unstructured":"Kirk Roberts, Dina Demner-Fushman, Ellen Voorhees, William R. Hersh, Steven Bedrick, Alexander J. Lazar, Shubham Pant, and Funda Meric-Bernstam. 2019. Overview of the TREC 2019 Precision Medicine Track. In TREC."},{"key":"e_1_3_2_1_68_1","volume-title":"Overview of the TREC 2016 Clinical Decision Support Track. In TREC.","author":"Roberts Kirk","unstructured":"Kirk Roberts, Dina Demner-Fushman, Ellen M. Voorhees, and William R. Hersh. 2016. Overview of the TREC 2016 Clinical Decision Support Track. In TREC."},{"key":"e_1_3_2_1_69_1","volume-title":"Overview of the TREC 2015 Clinical Decision Support Track. In TREC.","author":"Roberts Kirk","unstructured":"Kirk Roberts, Matthew S. Simpson, Ellen Voorhees, and William R. Hersh. 2015. Overview of the TREC 2015 Clinical Decision Support Track. In TREC."},{"key":"e_1_3_2_1_70_1","volume-title":"TREC Mandarin LDC2000T52","author":"Rogers Willie","year":"2000","unstructured":"Willie Rogers. 2000 a. TREC Mandarin LDC2000T52. https:\/\/catalog.ldc.upenn.edu\/LDC2000T52"},{"key":"e_1_3_2_1_71_1","volume-title":"TREC Spanish LDC2000T51","author":"Rogers Willie","year":"2000","unstructured":"Willie Rogers. 2000 b. TREC Spanish LDC2000T51. https:\/\/catalog.ldc.upenn.edu\/LDC2000T51"},{"key":"e_1_3_2_1_72_1","first-page":"e26752","article-title":"The new york times annotated corpus","volume":"6","author":"Sandhaus Evan","year":"2008","unstructured":"Evan Sandhaus. 2008. The new york times annotated corpus. Linguistic Data Consortium, Philadelphia, Vol. 6, 12 (2008), e26752.","journal-title":"Linguistic Data Consortium, Philadelphia"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"crossref","unstructured":"Royal Sequiera and Jimmy Lin. 2017. Finally a Downloadable Test Collection of Tweets. In SIGIR.","DOI":"10.1145\/3077136.3080667"},{"key":"e_1_3_2_1_74_1","volume-title":"Overview of the TREC 2014 Clinical Decision Support Track. In TREC.","author":"Simpson Matthew S.","year":"2014","unstructured":"Matthew S. Simpson, Ellen M. Voorhees, and William Hersh. 2014. Overview of the TREC 2014 Clinical Decision Support Track. In TREC."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"crossref","unstructured":"Alan Smeaton and Ross Wilkinson. 1996. Spanish and Chinese Document Retrieval in TREC-5. In TREC.","DOI":"10.6028\/NIST.SP.500-238.Chinese-overview"},{"key":"e_1_3_2_1_76_1","volume-title":"TREC 2018 News Track Overview. In TREC.","author":"Soboroff Ian","year":"2018","unstructured":"Ian Soboroff, Shudong Huang, and Donna Harman. 2018. TREC 2018 News Track Overview. In TREC."},{"key":"e_1_3_2_1_77_1","volume-title":"TREC 2019 News Track Overview. In TREC.","author":"Soboroff Ian","year":"2019","unstructured":"Ian Soboroff, Shudong Huang, and Donna Harman. 2019. TREC 2019 News Track Overview. In TREC."},{"key":"e_1_3_2_1_78_1","volume-title":"BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. arXiv preprint arXiv:2104.08663 (4","author":"Thakur Nandan","year":"2021","unstructured":"Nandan Thakur, Nils Reimers, Andreas R\u00fcckl\u00e9, Abhishek Srivastava, and Iryna Gurevych. 2021. BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. arXiv preprint arXiv:2104.08663 (4 2021). https:\/\/arxiv.org\/abs\/2104.08663"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_80_1","volume-title":"Overview of the TREC 2004 Robust Retrieval Track. In TREC.","author":"Voorhees Ellen","year":"2004","unstructured":"Ellen Voorhees. 2004. Overview of the TREC 2004 Robust Retrieval Track. In TREC."},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451965"},{"key":"e_1_3_2_1_82_1","volume-title":"Overview of the TREC 2005 Robust Retrieval Track. In TREC.","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees. 2005. Overview of the TREC 2005 Robust Retrieval Track. In TREC."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1023"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_85_1","unstructured":"Lucy Lu Wang Kyle Lo Yoganand Chandrasekhar Russell Reas Jiangjiang Yang Darrin Eide K. Funk Rodney Michael Kinney Ziyang Liu W. Merrill P. Mooney D. Murdick Devvret Rishi Jerry Sheehan Zhihong Shen B. Stilson A. Wade K. Wang Christopher Wilhelm Boya Xie D. Raymond Daniel S. Weld Oren Etzioni and Sebastian Kohlmeier. 2020. CORD-19: The Covid-19 Open Research Dataset. ArXiv (2020)."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"crossref","unstructured":"Ross Wilkinson. 1997. Chinese Document Retrieval at TREC-6. In TREC.","DOI":"10.6028\/NIST.SP.500-240.chinese-overview"},{"key":"e_1_3_2_1_87_1","unstructured":"Thomas Wolf Quentin Lhoest Patrick von Platen Yacine Jernite Mariama Drame Julien Plu Julien Chaumond Clement Delangue Clara Ma Abhishek Thakur Suraj Patil Joe Davison Teven Le Scao Victor Sanh Canwen Xu Nicolas Patry Angie McMillan-Major Simon Brandeis Sylvain Gugger Fran\u00e7ois Lagunas Lysandre Debut Morgan Funtowicz Anthony Moi Sasha Rush Philipp Schmidd Pierric Cistac Victor Mu?tar Jeff Boudier and Anna Tordjmann. 2020. Datasets. GitHub. Note: https:\/\/github.com\/huggingface\/datasets Vol. 1 (2020)."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080721"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412780"},{"key":"e_1_3_2_1_91_1","unstructured":"Guido Zuccon Joao Palotti Lorraine Goeuriot Liadh Kelly Mihai Lupu Pavel Pecina Henning M\u00fcller Julie Budaher and Anthony Deacon. 2016. The IR Task at the CLEF eHealth Evaluation Lab 2016: User-centred Health Information Retrieval. In CLEF."}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Virtual Event Canada","acronym":"SIGIR '21","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463254","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3463254","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:44Z","timestamp":1750191464000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463254"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":91,"alternative-id":["10.1145\/3404835.3463254","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3463254","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}