{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,12]],"date-time":"2026-06-12T02:28:43Z","timestamp":1781231323685,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Ahold Delhaize"},{"name":"Netherlands Organisation for Scientific Research"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3532686","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:08Z","timestamp":1657206728000},"page":"3459-3461","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Towards Reproducible Machine Learning Research in Information Retrieval"],"prefix":"10.1145","author":[{"given":"Ana","family":"Lucic","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maurits","family":"Bleeker","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Koustuv","family":"Sinha","sequence":"additional","affiliation":[{"name":"McGill University, Montreal, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sami","family":"Jullien","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Robert","family":"Stojnic","sequence":"additional","affiliation":[{"name":"Facebook AI Research, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"ACM. 2019. Artifact Review and Badging. https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging.  ACM. 2019. Artifact Review and Badging. https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.29"},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of the 14th International Conference on Natural Language Generation. Association for Computational Linguistics","author":"Belz Anya","year":"2021","unstructured":"Anya Belz , Anastasia Shimorina , Shubham Agarwal , and Ehud Reiter . 2021 . The ReproGen Shared Task on Reproducibility of Human Evaluations in NLG: Overview and Results . In Proceedings of the 14th International Conference on Natural Language Generation. Association for Computational Linguistics , Aberdeen, Scotland, UK, 249--258. Anya Belz, Anastasia Shimorina, Shubham Agarwal, and Ehud Reiter. 2021. The ReproGen Shared Task on Reproducibility of Human Evaluations in NLG: Overview and Results. In Proceedings of the 14th International Conference on Natural Language Generation. Association for Computational Linguistics, Aberdeen, Scotland, UK, 249--258."},{"key":"e_1_3_2_1_4_1","unstructured":"Alina Beygelzimer Yann Dauphin Percy Liang and Jennifer Wortman- Vaughan. 2021. Introducing the NeurIPS 2021 Paper Checklist. https:\/\/neuripsconf.medium.com\/introducing-the-neurips-2021-paperchecklist.  Alina Beygelzimer Yann Dauphin Percy Liang and Jennifer Wortman- Vaughan. 2021. Introducing the NeurIPS 2021 Paper Checklist. https:\/\/neuripsconf.medium.com\/introducing-the-neurips-2021-paperchecklist."},{"key":"e_1_3_2_1_5_1","volume-title":"International Conference on Machine Learning. PMLR, 725--734","author":"Bouthillier Xavier","year":"2019","unstructured":"Xavier Bouthillier , C\u00e9sar Laurent , and Pascal Vincent . 2019 . Unreproducible research is reproducible . In International Conference on Machine Learning. PMLR, 725--734 . Xavier Bouthillier, C\u00e9sar Laurent, and Pascal Vincent. 2019. Unreproducible research is reproducible. In International Conference on Machine Learning. PMLR, 725--734."},{"key":"e_1_3_2_1_6_1","volume-title":"Reproducible Online Search Experiments. In European Conference on Information Retrieval. Springer, 597--601","author":"Breuer Timo","year":"2020","unstructured":"Timo Breuer . 2020 . Reproducible Online Search Experiments. In European Conference on Information Retrieval. Springer, 597--601 . Timo Breuer. 2020. Reproducible Online Search Experiments. In European Conference on Information Retrieval. Springer, 597--601."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401036"},{"key":"e_1_3_2_1_8_1","volume-title":"The SIGIR 2019 Open-Source IR Replicability Challenge (OSIRRC","author":"Clancy Ryan","year":"2019","unstructured":"Ryan Clancy , Nicola Ferro , Claudia Hauff , Jimmy Lin , Tetsuya Sakai , and Ze Zhong Wu . 2019 . The SIGIR 2019 Open-Source IR Replicability Challenge (OSIRRC 2019). 1432--1434. Ryan Clancy, Nicola Ferro, Claudia Hauff, Jimmy Lin, Tetsuya Sakai, and Ze Zhong Wu. 2019. The SIGIR 2019 Open-Source IR Replicability Challenge (OSIRRC 2019). 1432--1434."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/275537.275544"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298689.3347058"},{"key":"e_1_3_2_1_11_1","volume-title":"Guest Post: Reproducibility at EMNLP","author":"Dodge Jesse","year":"2020","unstructured":"Jesse Dodge . 2020 . Guest Post: Reproducibility at EMNLP 2020. https:\/\/2020.emnlp.org\/blog\/2020-05--20-reproducibility. Jesse Dodge. 2020. Guest Post: Reproducibility at EMNLP 2020. https:\/\/2020.emnlp.org\/blog\/2020-05--20-reproducibility."},{"key":"e_1_3_2_1_12_1","unstructured":"Jesse Dodge. 2020. The Reproducibility Challenge as an Educational Tool. Medium https:\/\/medium.com\/paperswithcode\/the-reproducibility-challenge-asan-educational-tool-cd1596e3716c.  Jesse Dodge. 2020. The Reproducibility Challenge as an Educational Tool. Medium https:\/\/medium.com\/paperswithcode\/the-reproducibility-challenge-asan-educational-tool-cd1596e3716c."},{"key":"e_1_3_2_1_13_1","unstructured":"Chris Drummond. 2009. Replicability is not reproducibility: nor is it good science. (2009).  Chris Drummond. 2009. Replicability is not reproducibility: nor is it good science. (2009)."},{"key":"e_1_3_2_1_14_1","volume-title":"ECIR: Call for Reproducibility Track papers. https:\/\/www.ecir2021.eu\/call-for-reproducibility-track.","author":"ECIR.","year":"2021","unstructured":"ECIR. 2021 . ECIR: Call for Reproducibility Track papers. https:\/\/www.ecir2021.eu\/call-for-reproducibility-track. ECIR. 2021. ECIR: Call for Reproducibility Track papers. https:\/\/www.ecir2021.eu\/call-for-reproducibility-track."},{"key":"e_1_3_2_1_15_1","volume-title":"ECIR: Call for Reproducibility Track papers. https:\/\/ecir2022.org\/calls\/reproducibility\/.","author":"ECIR.","year":"2022","unstructured":"ECIR. 2022 . ECIR: Call for Reproducibility Track papers. https:\/\/ecir2022.org\/calls\/reproducibility\/. ECIR. 2022. ECIR: Call for Reproducibility Track papers. https:\/\/ecir2022.org\/calls\/reproducibility\/."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2964797.2964808"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274784.3274786"},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics","author":"Fokkens Antske","year":"2013","unstructured":"Antske Fokkens , Marieke van Erp , Marten Postma , Ted Pedersen , Piek Vossen , and Nuno Freire . 2013 . Offspring from Reproduction Problems: What Replication Failure Teaches Us . In Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics , Sofia, Bulgaria, 1691--1701. Antske Fokkens, Marieke van Erp, Marten Postma, Ted Pedersen, Piek Vossen, and Nuno Freire. 2013. Offspring from Reproduction Problems: What Replication Failure Teaches Us. In Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics, Sofia, Bulgaria, 1691--1701."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.359.6377.725"},{"key":"e_1_3_2_1_22_1","volume-title":"The relationship between documents read and relevant references retrieved as effectiveness measures for information retrieval systems. Journal of Documentation","author":"Lantz Brian E","year":"1981","unstructured":"Brian E Lantz . 1981. The relationship between documents read and relevant references retrieved as effectiveness measures for information retrieval systems. Journal of Documentation ( 1981 ). Brian E Lantz. 1981. The relationship between documents read and relevant references retrieved as effectiveness measures for information retrieval systems. Journal of Documentation (1981)."},{"key":"e_1_3_2_1_23_1","unstructured":"Ana Lucic. 2021. Case Study: How Your Course Can Incorporate the Reproducibility Challenge. Medium https:\/\/medium.com\/paperswithcode\/case-study-howyour-course-can-incorporate-the-reproducibility-challenge-76e260a2b59.  Ana Lucic. 2021. Case Study: How Your Course Can Incorporate the Reproducibility Challenge. Medium https:\/\/medium.com\/paperswithcode\/case-study-howyour-course-can-incorporate-the-reproducibility-challenge-76e260a2b59."},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the AAAI Symposium on Educational Advances in AI.","author":"Lucic Ana","year":"2021","unstructured":"Ana Lucic , Maurits Bleeker , Sami Jullien , Samarth Bhargav , and Maarten de Rijke . 2021 . Reproducibility as a Mechanism for Teaching Fairness, Accountability, Confidentiality, and Transparency in Artificial Intelligence . In Proceedings of the AAAI Symposium on Educational Advances in AI. Ana Lucic, Maurits Bleeker, Sami Jullien, Samarth Bhargav, and Maarten de Rijke. 2021. Reproducibility as a Mechanism for Teaching Fairness, Accountability, Confidentiality, and Transparency in Artificial Intelligence. In Proceedings of the AAAI Symposium on Educational Advances in AI."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313613"},{"key":"e_1_3_2_1_26_1","volume-title":"Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP","author":"Mieskes Margot","year":"2019","unstructured":"Margot Mieskes , Kar\u00ebn Fort , Aur\u00e9lie N\u00e9v\u00e9ol , Cyril Grouin , and Kevin Cohen . 2019 . Community Perspective on Replicability in Natural Language Processing . In Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP 2019). INCOMA Ltd., Varna, Bulgaria, 768--775. Margot Mieskes, Kar\u00ebn Fort, Aur\u00e9lie N\u00e9v\u00e9ol, Cyril Grouin, and Kevin Cohen. 2019. Community Perspective on Replicability in Natural Language Processing. In Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP 2019). INCOMA Ltd., Varna, Bulgaria, 768--775."},{"key":"e_1_3_2_1_27_1","volume-title":"Machine Learning Reproducibility Challenge","author":"MLRC.","year":"2021","unstructured":"MLRC. 2021. Machine Learning Reproducibility Challenge 2021 . https:\/\/paperswithcode.com\/rc2021. MLRC. 2021. Machine Learning Reproducibility Challenge 2021. https:\/\/paperswithcode.com\/rc2021."},{"key":"e_1_3_2_1_28_1","volume-title":"NAACL 2021 Reproducibility Checklist. https:\/\/2021","author":"NAACL.","year":"2021","unstructured":"NAACL. 2021 . NAACL 2021 Reproducibility Checklist. https:\/\/2021 .naacl.org\/calls\/reproducibility-checklist\/. NAACL. 2021. NAACL 2021 Reproducibility Checklist. https:\/\/2021.naacl.org\/calls\/reproducibility-checklist\/."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli.2008.34.3.465"},{"key":"e_1_3_2_1_30_1","volume-title":"manual_seed (3407) is all you need: On the influence of random seeds in deep learning architectures for computer vision. arXiv preprint arXiv:2109.08203","author":"Picard David","year":"2021","unstructured":"David Picard . 2021. Torch. manual_seed (3407) is all you need: On the influence of random seeds in deep learning architectures for computer vision. arXiv preprint arXiv:2109.08203 ( 2021 ). David Picard. 2021. Torch. manual_seed (3407) is all you need: On the influence of random seeds in deep learning architectures for computer vision. arXiv preprint arXiv:2109.08203 (2021)."},{"key":"e_1_3_2_1_31_1","volume-title":"Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program). Journal of Machine Learning Research","author":"Pineau Joelle","year":"2020","unstructured":"Joelle Pineau , Philippe Vincent-Lamarre , Koustuv Sinha , Vincent Lariviere , Alina Beygelzimer , Florence d' Alche Buc , Emily Fox , and Hugo Larochelle . 2020. Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program). Journal of Machine Learning Research ( 2020 ). Joelle Pineau, Philippe Vincent-Lamarre, Koustuv Sinha, Vincent Lariviere, Alina Beygelzimer, Florence d'Alche Buc, Emily Fox, and Hugo Larochelle. 2020. Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program). Journal of Machine Learning Research (2020)."},{"key":"e_1_3_2_1_32_1","first-page":"5485","article-title":"A step toward quantifying independently reproducible machine learning research","volume":"32","author":"Raff Edward","year":"2019","unstructured":"Edward Raff . 2019 . A step toward quantifying independently reproducible machine learning research . Advances in Neural Information Processing Systems 32 (2019), 5485 -- 5495 . Edward Raff. 2019. A step toward quantifying independently reproducible machine learning research. Advances in Neural Information Processing Systems 32 (2019), 5485--5495.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_33_1","volume-title":"SIGIR 2022 Call for Reproducibility Track Papers. https:\/\/sigir.org\/sigir2022\/call-for-reproducibility-track-papers\/.","author":"SIGIR.","year":"2022","unstructured":"SIGIR. 2022 . SIGIR 2022 Call for Reproducibility Track Papers. https:\/\/sigir.org\/sigir2022\/call-for-reproducibility-track-papers\/. SIGIR. 2022. SIGIR 2022 Call for Reproducibility Track Papers. https:\/\/sigir.org\/sigir2022\/call-for-reproducibility-track-papers\/."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00330"},{"key":"e_1_3_2_1_35_1","volume-title":"Generalizability and Reproducibility of Search Engine Online User Studies. Master's thesis","author":"Zijan Xu.","unstructured":"Zijan Xu. 2020. Generalizability and Reproducibility of Search Engine Online User Studies. Master's thesis . Virginia Polytechnic Institute and State University . Zijan Xu. 2020. Generalizability and Reproducibility of Search Engine Online User Studies. Master's thesis. Virginia Polytechnic Institute and State University."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-76423-4_1"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Madrid Spain","acronym":"SIGIR '22","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532686","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3532686","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:36Z","timestamp":1750183836000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3532686"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":36,"alternative-id":["10.1145\/3477495.3532686","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3532686","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}