{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,28]],"date-time":"2026-03-28T03:41:57Z","timestamp":1774669317471,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657884","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"293-302","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["CIRAL: A Test Collection for CLIR Evaluations in African Languages"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-2859-7136","authenticated-orcid":false,"given":"Mofetoluwa","family":"Adeyemi","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2630-8167","authenticated-orcid":false,"given":"Akintunde","family":"Oladipo","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0756-8110","authenticated-orcid":false,"given":"Xinyu","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-4591-3077","authenticated-orcid":false,"given":"David","family":"Alfonso-Hermelo","sequence":"additional","affiliation":[{"name":"Huawei Technologies, Montreal, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4014-6007","authenticated-orcid":false,"given":"Mehdi","family":"Rezagholizadeh","sequence":"additional","affiliation":[{"name":"Huawei Technologies, Montreal, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3170-4858","authenticated-orcid":false,"given":"Boxing","family":"Chen","sequence":"additional","affiliation":[{"name":"Huawei Technologies, Montreal, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-4558-0356","authenticated-orcid":false,"given":"Abdul-Hakeem","family":"Omotayo","sequence":"additional","affiliation":[{"name":"University of California, Davis, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3795-8381","authenticated-orcid":false,"given":"Idris","family":"Abdulmumin","sequence":"additional","affiliation":[{"name":"University of Pretoria, Pretoria, South Africa"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7772-1103","authenticated-orcid":false,"given":"Naome A.","family":"Etori","sequence":"additional","affiliation":[{"name":"University of Minnesota, Twin Cities, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0239-6416","authenticated-orcid":false,"given":"Toyib Babatunde","family":"Musa","sequence":"additional","affiliation":[{"name":"Masakhane, Lagos, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4636-1437","authenticated-orcid":false,"given":"Samuel","family":"Fanijo","sequence":"additional","affiliation":[{"name":"Iowa State University, Ames, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5563-8987","authenticated-orcid":false,"given":"Oluwabusayo Olufunke","family":"Awoyomi","sequence":"additional","affiliation":[{"name":"The College of Saint Rose, Albany, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1458-4057","authenticated-orcid":false,"given":"Saheed Abdullahi","family":"Salahudeen","sequence":"additional","affiliation":[{"name":"Shenzhen Institute of Advanced Technology, CAS, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9090-5072","authenticated-orcid":false,"given":"Labaran Adamu","family":"Mohammed","sequence":"additional","affiliation":[{"name":"Kwame Nkrumah University of Science and Technology, Kumasi, Ghana"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0090-9486","authenticated-orcid":false,"given":"Daud Olamide","family":"Abolade","sequence":"additional","affiliation":[{"name":"Masakhane, Lagos, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3310-0326","authenticated-orcid":false,"given":"Falalu Ibrahim","family":"Lawan","sequence":"additional","affiliation":[{"name":"Masakhane, Kano, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-0379-2256","authenticated-orcid":false,"given":"Maryam","family":"Sabo Abubakar","sequence":"additional","affiliation":[{"name":"Masakhane, Kano, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3885-8488","authenticated-orcid":false,"given":"Ruqayya","family":"Nasir Iro","sequence":"additional","affiliation":[{"name":"Masakhane, Kano, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0985-1404","authenticated-orcid":false,"given":"Amina","family":"Imam Abubakar","sequence":"additional","affiliation":[{"name":"University of Abuja, Abuja, Nigeria"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-2844-5304","authenticated-orcid":false,"given":"Shafie Abdi","family":"Mohamed","sequence":"additional","affiliation":[{"name":"Masakhane, Mogadishu, Somalia"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8423-6079","authenticated-orcid":false,"given":"Hanad Mohamud","family":"Mohamed","sequence":"additional","affiliation":[{"name":"Masakhane, Mogadishu, Somalia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1860-609X","authenticated-orcid":false,"given":"Tunde Oluwaseyi","family":"Ajayi","sequence":"additional","affiliation":[{"name":"University of Galway, Galway, Ireland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0661-7189","authenticated-orcid":false,"given":"Jimmy","family":"Lin","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 3053--3070","author":"David","unstructured":"David Adelani et al. 2022. A Few Thousand Translations Go a Long Way! Leveraging Pre-trained Models for African News Translation. In Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 3053--3070."},{"key":"e_1_3_2_1_2_1","unstructured":"David Adelani et al. 2023. MasakhaNews: News Topic Classification for African languages. arXiv:2304.09972 (2023)."},{"key":"e_1_3_2_1_3_1","volume-title":"Spacerini: Plug-and-Play Search Engines with Pyserini and Hugging Face. arXiv:2302.14534","author":"Akiki Christopher","year":"2023","unstructured":"Christopher Akiki, Odunayo Ogundepo, Aleksandra Piktus, Xinyu Zhang, Akintunde Oladipo, Jimmy Lin, and Martin Potthast. 2023. Spacerini: Plug-and-Play Search Engines with Pyserini and Hugging Face. arXiv:2302.14534 (2023)."},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 29th International Conference on Computational Linguistics. 4336--4349","author":"Alabi Jesujoba O.","year":"2022","unstructured":"Jesujoba O. Alabi, David Ifeoluwa Adelani, Marius Mosbach, and Dietrich Klakow. 2022. Adapting Pre-trained Language Models to African Languages via Multilingual Adaptive Fine-Tuning. In Proceedings of the 29th International Conference on Computational Linguistics. 4336--4349."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1093\/llc\/fqu047"},{"key":"e_1_3_2_1_6_1","volume-title":"MS MARCO: A Human Generated Machine Reading Comprehension Dataset. arXiv:1611.09268","author":"Bajaj Payal","year":"2016","unstructured":"Payal Bajaj, Daniel Campos, Nick Craswell, Li Deng, Jianfeng Gao, Xiaodong Liu, Rangan Majumder, Andrew McNamara, Bhaskar Mitra, Tri Nguyen, et al. 2016. MS MARCO: A Human Generated Machine Reading Comprehension Dataset. arXiv:1611.09268 (2016)."},{"key":"e_1_3_2_1_7_1","volume-title":"Israel Campiotti, Marzieh Fadaee, Roberto Lotufo, and Rodrigo Nogueira.","author":"Bonifacio Luiz","year":"2021","unstructured":"Luiz Bonifacio, Vitor Jeronymo, Hugo Queiroz Abonizio, Israel Campiotti, Marzieh Fadaee, Roberto Lotufo, and Rodrigo Nogueira. 2021. mMARCO: A Multilingual Version of the MS MARCO Passage Ranking Dataset. arXiv:2108.13897 (2021)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00317"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572114"},{"key":"e_1_3_2_1_11_1","volume-title":"Costa-juss\u00e0 et al","author":"Marta","year":"2022","unstructured":"Marta R. Costa-juss\u00e0 et al. 2022. No Language Left Behind: Scaling Human-Centered Machine Translation. arXiv:2207.04672 (2022)."},{"key":"e_1_3_2_1_12_1","volume-title":"Overview of the TREC 2022 Deep Learning Track. In Text REtrieval Conference (TREC).","author":"Craswell Nick","year":"2023","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, Jimmy Lin, Ellen M. Voorhees, and Ian Soboroff. 2023. Overview of the TREC 2022 Deep Learning Track. In Text REtrieval Conference (TREC)."},{"key":"e_1_3_2_1_13_1","volume-title":"Overview of the TREC 2019 Deep Learning Track. arXiv:2003","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M. Voorhees. 2020. Overview of the TREC 2019 Deep Learning Track. arXiv:2003.07820 (2020)."},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4171--4186","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4171--4186."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Nicola Ferro and Carol Peters. 2019. Information Retrieval Evaluation in a Changing World: Lessons Learned from 20 Years of CLEF.","DOI":"10.1007\/978-3-030-22948-1"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463098"},{"key":"e_1_3_2_1_17_1","volume-title":"Park","author":"Gaim Fitsum","year":"2023","unstructured":"Fitsum Gaim, Wonsuk Yang, Hancheol Park, and Jong C. Park. 2023. Question-Answering in a Low-resourced Language: Benchmark Dataset and Models for Tigrinya. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics. 11857--11870."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the First NTCIR Workshop. 11--44","author":"Kando Noriko","year":"1999","unstructured":"Noriko Kando, Kazuko Kuriyama, Toshihiko Nozue, Koji Eguchi, Hiroyuki Kato, and Souichiro Hidaka. 1999. Overview of IR Tasks at the First NTCIR Workshop. In Proceedings of the First NTCIR Workshop. 11--44."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1162\/COLI_a_00111"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463238"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1838745.1838747"},{"key":"e_1_3_2_1_23_1","volume-title":"Transfer Learning Approaches for Building Cross-language Dense Retrieval Models. In European Conference on Information Retrieval. 382--396","author":"Nair Suraj","unstructured":"Suraj Nair, Eugene Yang, Dawn Lawrie, Kevin Duh, Paul McNamee, Kenton Murray, James Mayfield, and Douglas W. Oard. 2022. Transfer Learning Approaches for Building Cross-language Dense Retrieval Models. In European Conference on Information Retrieval. 382--396."},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1219--1229","author":"Nair Suraj","unstructured":"Suraj Nair, Eugene Yang, Dawn Lawrie, James Mayfield, and Douglas W. Oard. 2023. BLADE: Combining Vocabulary Pruning and Intermediate Pretraining for Scaleable Neural CLIR. In Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1219--1229."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.mrl-1.11"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.597"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.deeplo-1.14"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00148"},{"key":"e_1_3_2_1_30_1","volume-title":"100,000 Questions for Machine Comprehension of Text. arXiv:1606.05250","author":"Rajpurkar Pranav","year":"2016","unstructured":"Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, and Percy Liang. 2016. SQuAD: 100,000 Questions for Machine Comprehension of Text. arXiv:1606.05250 (2016)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_1_32_1","volume-title":"Conferences of the Association for Machine Translation in the Americas: MT Users' Track. 322--354","author":"Rubino Carl","year":"2016","unstructured":"Carl Rubino. 2016. Machine Translation for English Retrieval of Information in any Language (Machine Translation for English-based Domain-Appropriate Triage of Information in any Language). In Conferences of the Association for Machine Translation in the Americas: MT Users' Track. 322--354."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557325"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-2073"},{"key":"e_1_3_2_1_35_1","volume-title":"Cross-Language Information Retrieval (CLIR) Track Overview. In Text REtrieval Conference (TREC).","author":"Peter","year":"1997","unstructured":"Peter Sch\"auble. 1997. Cross-Language Information Retrieval (CLIR) Track Overview. In Text REtrieval Conference (TREC)."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.340"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-28241-6_11"},{"key":"e_1_3_2_1_38_1","volume-title":"BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. arXiv:2104.08663","author":"Thakur Nandan","year":"2021","unstructured":"Nandan Thakur, Nils Reimers, Andreas R\u00fcckl\u00e9, Abhishek Srivastava, and Iryna Gurevych. 2021. BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. arXiv:2104.08663 (2021)."},{"key":"e_1_3_2_1_39_1","first-page":"360","article-title":"Understanding Interobserver Agreement: The Kappa Statistic","volume":"37","author":"Viera Anthony J.","year":"2005","unstructured":"Anthony J. Viera and Joanne M. Garrett. 2005. Understanding Interobserver Agreement: The Kappa Statistic. Family Medicine, Vol. 37, 5 (2005), 360--363.","journal-title":"Family Medicine"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1093\/llc\/fqt031"},{"key":"e_1_3_2_1_41_1","volume-title":"mT5: A Massively Multilingual Pre-trained Text-to-Text Transformer. arXiv:2010.11934","author":"Xue Linting","year":"2020","unstructured":"Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, and Colin Raffel. 2020. mT5: A Massively Multilingual Pre-trained Text-to-Text Transformer. arXiv:2010.11934 (2020)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080721"},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of the Workshop on Cross-Language Search and Summarization of Text and Speech (CLSSTS2020)","author":"Zavorin Ilya","year":"2020","unstructured":"Ilya Zavorin, Aric Bills, Cassian Corey, Michelle Morrison, Audrey Tong, and Richard Tong. 2020. Corpora for Cross-language Information Retrieval in Six Less-Resourced Languages. In Proceedings of the Workshop on Cross-Language Search and Summarization of Text and Speech (CLSSTS2020). 7--13."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.mrl-1.12"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00595"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657884","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657884","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:32:58Z","timestamp":1755840778000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657884"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":46,"alternative-id":["10.1145\/3626772.3657884","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657884","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}