{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T22:37:51Z","timestamp":1785537471340,"version":"3.56.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"The Center for Research and Education in AI and Learning (REAL@USC)"},{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["OAC 2138773"],"award-info":[{"award-number":["OAC 2138773"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663629","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"101-109","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Reproscreener: Leveraging LLMs for Assessing Computational Reproducibility of Machine Learning Pipelines"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4349-105X","authenticated-orcid":false,"given":"Adhithya","family":"Bhaskar","sequence":"first","affiliation":[{"name":"University of Southern California, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2015-7825","authenticated-orcid":false,"given":"Victoria","family":"Stodden","sequence":"additional","affiliation":[{"name":"University of Southern California, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Sharing high expectations. 1, 8","year":"2019","unstructured":"2019. Sharing high expectations. 1, 8 (2019), 329\u2013329. Number: 8 Publisher: Nature Publishing Group."},{"key":"e_1_3_2_1_2_1","volume-title":"Reproducibility of research: Issues and proposed remedies. 115, 11","author":"Allison B.","year":"2018","unstructured":"David\u00a0B. Allison, Richard\u00a0M. Shiffrin, and Victoria Stodden. 2018. Reproducibility of research: Issues and proposed remedies. 115, 11 (2018), 2561\u20132562. Publisher: Proceedings of the National Academy of Sciences."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Iz Beltagy Kyle Lo and Arman Cohan. 2019. SciBERT: A Pretrained Language Model for Scientific Text. arxiv:1903.10676 [cs]","DOI":"10.18653\/v1\/D19-1371"},{"key":"e_1_3_2_1_4_1","unstructured":"Adhithya Bhaskar and Victoria Stodden. 2024. Reproscreener."},{"key":"e_1_3_2_1_5_1","volume-title":"Using artificial intelligence methods for systematic review in health sciences: A systematic review. 13, 3","author":"Blaizot Aymeric","year":"2022","unstructured":"Aymeric Blaizot, Sajesh\u00a0K. Veettil, Pantakarn Saidoung, Carlos\u00a0Francisco Moreno-Garcia, Nirmalie Wiratunga, Magaly Aceves-Martins, Nai\u00a0Ming Lai, and Nathorn Chaiyakunapruk. 2022. Using artificial intelligence methods for systematic review in health sciences: A systematic review. 13, 3 (2022), 353\u2013362."},{"key":"e_1_3_2_1_6_1","volume-title":"Machine Learning in Weather Prediction and Climate Analyses\u2014Applications and Perspectives. 13, 2","author":"Bochenek Bogdan","year":"2022","unstructured":"Bogdan Bochenek and Zbigniew Ustrnul. 2022. Machine Learning in Weather Prediction and Climate Analyses\u2014Applications and Perspectives. 13, 2 (2022), 180. Number: 2 Publisher: Multidisciplinary Digital Publishing Institute."},{"key":"e_1_3_2_1_7_1","volume-title":"\u201cWhole Tale\u201d. 94","author":"Brinckman Adam","year":"2019","unstructured":"Adam Brinckman, Kyle Chard, Niall Gaffney, Mihael Hategan, Matthew\u00a0B. Jones, Kacper Kowalik, Sivakumar Kulasekaran, Bertram Lud\u00e4scher, Bryce\u00a0D. Mecum, Jarek Nabrzyski, Victoria Stodden, Ian\u00a0J. Taylor, Matthew\u00a0J. Turk, and Kandace Turner. 2019. Computing environments for reproducibility: Capturing the \u201cWhole Tale\u201d. 94 (2019), 854\u2013867."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Merc\u00e8 Crosas Julian Gautier Sebastian Karcher Dessi Kirilova Gerard Otalora and Abigail Schwartz. 2018. Data policies of highly-ranked social science journals.","DOI":"10.31235\/osf.io\/9h7ay"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1080\/0952813X.2019.1658857"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Paolo Di\u00a0Tommaso Maria Chatzou Evan Floden Pablo Prieto\u00a0Barja Emilio Palumbo and Cedric Notredame. 2017. Nextflow enables reproducible computational workflows.","DOI":"10.1038\/nbt.3820"},{"key":"e_1_3_2_1_11_1","volume-title":"State of the Art: Reproducibility in Artificial Intelligence. 32, 1","author":"Gundersen Odd\u00a0Erik","year":"2018","unstructured":"Odd\u00a0Erik Gundersen and Sigbj\u00f8rn Kjensmo. 2018. State of the Art: Reproducibility in Artificial Intelligence. 32, 1 (2018). Section: AAAI Technical Track: Humans and AI."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1038\/s43705-022-00182-9"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3422308"},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of the 2020 USENIX Annual Technical Conference (USENIX ATC \u201920)","author":"Keahey Kate","year":"2020","unstructured":"Kate Keahey, Jason Anderson, Zhuo Zhen, Pierre Riteau, Paul Ruth, Dan Stanzione, Mert Cevik, Jacob Colleran, Haryadi\u00a0S. Gunawi, Cody Hammock, Joe Mambretti, Alexander Barnes, Fran\u00e7ois Halbach, Alex Rocha, and Joe Stubbs. 2020. Lessons Learned from the Chameleon Testbed. In Proceedings of the 2020 USENIX Annual Technical Conference (USENIX ATC \u201920). USENIX Association."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"M.\u00a0S. Krafczyk A. Shi A. Bhaskar D. Marinov and V. Stodden. 2021. Learning from reproducing computational results: introducing three principles and the Reproduction Package. 379 2197 (2021) rsta.2020.0069 20200069.","DOI":"10.1098\/rsta.2020.0069"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Engineering National Academies\u00a0of Sciences. 2019. Reproducibility and Replicability in Science.","DOI":"10.1162\/99608f92.cb310198"},{"key":"e_1_3_2_1_17_1","volume-title":"Publisher: American Association for the Advancement of Science.","author":"Nosek A.","year":"2015","unstructured":"B.\u00a0A. Nosek, G. Alter, G.\u00a0C. Banks, D. Borsboom, S.\u00a0D. Bowman, S.\u00a0J. Breckler, S. Buck, C.\u00a0D. Chambers, G. Chin, G. Christensen, M. Contestabile, A. Dafoe, E. Eich, J. Freese, R. Glennerster, D. Goroff, D.\u00a0P. Green, B. Hesse, M. Humphreys, J. Ishiyama, D. Karlan, A. Kraut, A. Lupia, P. Mabry, T. Madon, N. Malhotra, E. Mayo-Wilson, M. McNutt, E. Miguel, E.\u00a0Levy Paluck, U. Simonsohn, C. Soderberg, B.\u00a0A. Spellman, J. Turitto, G. VandenBos, S. Vazire, E.\u00a0J. Wagenmakers, R. Wilson, and T. Yarkoni. 2015. Promoting an open research culture. 348, 6242 (2015), 1422\u20131425. Publisher: American Association for the Advancement of Science."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP53844.2022.00048"},{"key":"e_1_3_2_1_19_1","unstructured":"Joelle Pineau Philippe Vincent-Lamarre Koustuv Sinha Vincent Larivi\u00e8re Alina Beygelzimer Florence d\u2019Alch\u00e9 Buc Emily Fox and Hugo Larochelle. 2020. Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program). arxiv:2003.12206 [cs stat]"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.25080\/Majora-4af1f417-011"},{"key":"e_1_3_2_1_21_1","volume-title":"Machine learning-based approach: global trends, research directions, and regulatory standpoints. 4","author":"Pugliese Raffaele","year":"2021","unstructured":"Raffaele Pugliese, Stefano Regondi, and Riccardo Marini. 2021. Machine learning-based approach: global trends, research directions, and regulatory standpoints. 4 (2021), 19\u201329."},{"key":"e_1_3_2_1_22_1","first-page":"29207","article-title":"Reproducibility in Computing Research","volume":"10","author":"Raghupathi Wullianallur","year":"2022","unstructured":"Wullianallur Raghupathi, Viju Raghupathi, and Jie Ren. 2022. Reproducibility in Computing Research: An Empirical Study. 10 (2022), 29207\u201329223.","journal-title":"An Empirical Study."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-80960-7_17"},{"key":"e_1_3_2_1_24_1","first-page":"160","article-title":"Machine Learning: Algorithms","volume":"2","author":"Sarker H.","year":"2021","unstructured":"Iqbal\u00a0H. Sarker. 2021. Machine Learning: Algorithms, Real-World Applications and Research Directions. 2, 3 (2021), 160.","journal-title":"Real-World Applications and Research Directions."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Amanpreet Singh Mike D\u2019Arcy Arman Cohan Doug Downey and Sergey Feldman. 2023. SciRepEval: A Multi-Format Benchmark for Scientific Document Representations. arxiv:2211.13308 [cs]","DOI":"10.18653\/v1\/2023.emnlp-main.338"},{"key":"e_1_3_2_1_26_1","unstructured":"Victoria Stodden. [n. d.]. Enabling Reproducible Research: Open Licensing for Scientific Innovation."},{"key":"e_1_3_2_1_27_1","volume-title":"Toward Reproducible Computational Research: An Empirical Analysis of Data and Code Policy Adoption by Journals. PLOS ONE 8, 6 (06","author":"Stodden Victoria","year":"2013","unstructured":"Victoria Stodden, Peixuan Guo, and Zhaokun Ma. 2013. Toward Reproducible Computational Research: An Empirical Analysis of Data and Code Policy Adoption by Journals. PLOS ONE 8, 6 (06 2013), 1\u20138."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3214239.3214242"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1708290115"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1080\/10494820.2021.1875001"},{"key":"e_1_3_2_1_31_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian\u00a0Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit\u00a0Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric\u00a0Michael Smith Ranjan Subramanian Xiaoqing\u00a0Ellen Tan Binh Tang Ross Taylor Adina Williams Jian\u00a0Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arxiv:2307.09288\u00a0[cs]"},{"key":"e_1_3_2_1_32_1","volume-title":"Erik van Mulligen, Jan Velterop, Andra Waagmeester, Peter Wittenburg, Katherine Wolstencroft, Jun Zhao, and Barend Mons.","author":"Wilkinson D.","year":"2016","unstructured":"Mark\u00a0D. Wilkinson, Michel Dumontier, IJsbrand\u00a0Jan Aalbersberg, Gabrielle Appleton, Myles Axton, Arie Baak, Niklas Blomberg, Jan-Willem Boiten, Luiz\u00a0Bonino da Silva\u00a0Santos, Philip\u00a0E. Bourne, Jildau Bouwman, Anthony\u00a0J. Brookes, Tim Clark, Merc\u00e8 Crosas, Ingrid Dillo, Olivier Dumon, Scott Edmunds, Chris\u00a0T. Evelo, Richard Finkers, Alejandra Gonzalez-Beltran, Alasdair J.\u00a0G. Gray, Paul Groth, Carole Goble, Jeffrey\u00a0S. Grethe, Jaap Heringa, Peter A.\u00a0C. \u2019t Hoen, Rob Hooft, Tobias Kuhn, Ruben Kok, Joost Kok, Scott\u00a0J. Lusher, Maryann\u00a0E. Martone, Albert Mons, Abel\u00a0L. Packer, Bengt Persson, Philippe Rocca-Serra, Marco Roos, Rene van Schaik, Susanna-Assunta Sansone, Erik Schultes, Thierry Sengstag, Ted Slater, George Strawn, Morris\u00a0A. Swertz, Mark Thompson, Johan van\u00a0der Lei, Erik van Mulligen, Jan Velterop, Andra Waagmeester, Peter Wittenburg, Katherine Wolstencroft, Jun Zhao, and Barend Mons. 2016. The FAIR Guiding Principles for scientific data management and stewardship. 3, 1 (2016), 160018. Number: 1 Publisher: Nature Publishing Group."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3589806.3600039"},{"key":"e_1_3_2_1_34_1","first-page":"e27850","article-title":"Chatbot for Health Care and Oncology Applications Using Artificial Intelligence and Machine Learning","volume":"7","author":"Xu Lu","year":"2021","unstructured":"Lu Xu, Leslie Sanders, Kay Li, and James C.\u00a0L. Chow. 2021. Chatbot for Health Care and Oncology Applications Using Artificial Intelligence and Machine Learning: Systematic Review. 7, 4 (2021), e27850.","journal-title":"Systematic Review."},{"key":"e_1_3_2_1_35_1","unstructured":"Shenghao Xu. 2022. BanditMF: Multi-Armed Bandit Based Matrix Factorization Recommender System. arxiv:2106.10898 [cs]"}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","location":"Rennes France","acronym":"ACM REP '24","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663629","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663629","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:04:35Z","timestamp":1755914675000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663629"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":35,"alternative-id":["10.1145\/3641525.3663629","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663629","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}