{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T07:59:58Z","timestamp":1776931198102,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,29]]},"DOI":"10.1145\/3736731.3746157","type":"proceedings-article","created":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T11:58:19Z","timestamp":1761047899000},"page":"234-242","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Raising the Reproducibility Bar"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4024-239X","authenticated-orcid":false,"given":"Joseph","family":"Wonsil","sequence":"first","affiliation":[{"name":"University of British Columbia, Vancouver, British Columbia, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8601-8441","authenticated-orcid":false,"given":"R\u00fabia","family":"Guerra","sequence":"additional","affiliation":[{"name":"University of British Columbia, Vancouver, British Columbia, Canada"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2449-0844","authenticated-orcid":false,"given":"Adam","family":"Pocock","sequence":"additional","affiliation":[{"name":"Oracle Labs, Burlington, Massachusetts, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-2877-401X","authenticated-orcid":false,"given":"Jack","family":"Sullivan","sequence":"additional","affiliation":[{"name":"Oracle Labs, Burlington, Massachusetts, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2165-4658","authenticated-orcid":false,"given":"Margo","family":"Seltzer","sequence":"additional","affiliation":[{"name":"University of British Columbia, Vancouver, British Columbia, Canada"}]}],"member":"320","published-online":{"date-parts":[[2025,10,21]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"2025. DeepWiki. deepwiki.com."},{"key":"e_1_3_3_2_3_2","unstructured":"2025. Papers with Code. https:\/\/portal.paperswithcode.com\/."},{"key":"e_1_3_3_2_4_2","volume-title":"A Taxonomy for Learning, Teaching, and Assessing: A Revision of Bloom\u2019s Taxonomy of Educational Objectives","author":"Anderson Lorin\u00a0W.","year":"2001","unstructured":"Lorin\u00a0W. Anderson and David\u00a0R. Krathwohl. 2001. A Taxonomy for Learning, Teaching, and Assessing: A Revision of Bloom\u2019s Taxonomy of Educational Objectives. Longman."},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","unstructured":"Monya Baker. 2016. 1 500 Scientists Lift the Lid on Reproducibility. Nature 533 7604 (May 2016) 452\u2013454. 10.1038\/533452a","DOI":"10.1038\/533452a"},{"key":"e_1_3_3_2_6_2","volume-title":"Taxonomy of Educational Objectives: The Classification of Educational Goals. Handbook 1: Cognitive Domain","author":"Bloom Benjamin\u00a0S","year":"1956","unstructured":"Benjamin\u00a0S Bloom, Max\u00a0D Engelhart, Edward\u00a0J Furst, Walker\u00a0H Hill, and David\u00a0R Krathwohl. 1956. Taxonomy of Educational Objectives: The Classification of Educational Goals. Handbook 1: Cognitive Domain. Longman New York."},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641525.3663617"},{"key":"e_1_3_3_2_8_2","unstructured":"Center for OpenScience. 2025. Open Science Badges. https:\/\/www.cos.io\/initiatives\/badges."},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","unstructured":"Eitan Frachtenberg. 2022. Research Artifacts and Citations in Computer Systems Papers. PeerJ Computer Science 8 (Feb. 2022) e887. 10.7717\/peerj-cs.887","DOI":"10.7717\/peerj-cs.887"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Ann Gabriel and Rebecca Capone. 2011. Executable Paper Grand Challenge Workshop. Procedia Computer Science 4 (2011) 577\u2013578. 10.1016\/j.procs.2011.04.060","DOI":"10.1016\/j.procs.2011.04.060"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3456287.3465478"},{"key":"e_1_3_3_2_12_2","first-page":"7","volume-title":"4th USENIX Workshop on the Theory and Practice of Provenance (TaPP 12)","author":"Guo Philip\u00a0J.","year":"2012","unstructured":"Philip\u00a0J. Guo and Margo Seltzer. 2012. {BURRITO}: Wrapping Your Lab Notebook in Computational Infrastructure. In 4th USENIX Workshop on the Theory and Practice of Provenance (TaPP 12). 7\u20137."},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380366"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","unstructured":"M.\u00a0S. Krafczyk A. Shi A. Bhaskar D. Marinov and V. Stodden. 2021. Learning from Reproducing Computational Results: Introducing Three Principles and the Reproduction Package. Philosophical Transactions of the Royal Society A: Mathematical Physical and Engineering Sciences 379 2197 (March 2021) 20200069. 10.1098\/rsta.2020.0069","DOI":"10.1098\/rsta.2020.0069"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","unstructured":"Barbara Lerner Emery Boose Orenna Brand Aaron\u00a0M. Ellison Elizabeth Fong Matthew Lau Khanh Ngo Thomas Pasquier Luis\u00a0A. Perez Margo Seltzer Rose Sheehan and Joseph Wonsil. 2023. Making Provenance Work for You. The R Journal 14 4 (Feb. 2023) 141\u2013159. 10.32614\/RJ-2023-003","DOI":"10.32614\/RJ-2023-003"},{"key":"e_1_3_3_2_16_2","volume-title":"How Learning Works: Eight Research-Based Principles for Smart Teaching","author":"Lovett Marsha\u00a0C.","year":"2023","unstructured":"Marsha\u00a0C. Lovett, Michael\u00a0W. Bridges, Michele DiPietro, Susan\u00a0A. Ambrose, and Marie\u00a0K. Norman. 2023. How Learning Works: Eight Research-Based Principles for Smart Teaching. John Wiley & Sons, Incorporated, Newark, UNITED STATES."},{"key":"e_1_3_3_2_17_2","unstructured":"Alexandre\u00a0Zouaoui Michael\u00a0Arbel. 2024. MLXP: A Framework for Conducting Replicable Machine Learning eXperiments in Python. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.13831."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-16462-56"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.17226\/25303"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","unstructured":"Peter Naur. 1985. Programming as Theory Building. Microprocessing and Microprogramming 15 5 (May 1985) 253\u2013261. 10.1016\/0165-6074(85)90032-8","DOI":"10.1016\/0165-6074(85)90032-8"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641525.3663628"},{"key":"e_1_3_3_2_22_2","unstructured":"Perplexity. 2023. Perplexity.Ai (AI Chatbot) [Large Language Model]. https:\/\/www.perplexity.ai\/."},{"key":"e_1_3_3_2_23_2","unstructured":"Adam Pocock. 2021. Tribuo: Machine Learning with Provenance in Java. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2110.03022 (2021). arxiv:https:\/\/arXiv.org\/abs\/2110.03022"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3589806.3600041"},{"key":"e_1_3_3_2_25_2","volume-title":"The Concept of Mind","author":"Ryle Gilbert","year":"1949","unstructured":"Gilbert Ryle. 1949. The Concept of Mind. Hutchinson London."},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","unstructured":"Evanthia\u00a0Kaimaklioti Samota and Robert\u00a0P. Davey. 2021. Knowledge and Attitudes Among Life Scientists Toward Reproducibility Within Journal Articles: A Research Survey. Frontiers in Research Metrics and Analytics 6 (June 2021). 10.3389\/frma.2021.678554","DOI":"10.3389\/frma.2021.678554"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","unstructured":"Sheeba Samuel and Birgitta K\u00f6nig-Ries. 2021. Understanding Experiments and Research Practices for Reproducibility: An Exploratory Study. PeerJ 9 (April 2021) e11140. 10.7717\/peerj.11140","DOI":"10.7717\/peerj.11140"},{"key":"e_1_3_3_2_28_2","unstructured":"Giulio Starace Oliver Jaffe Dane Sherburn James Aung Chan\u00a0Jun Shern Leon Maksin Rachel Dias Evan Mays Benjamin Kinsella Wyatt Thompson Johannes Heidecke Amelia Glaese and Tejal Patwardhan. 2025. PaperBench: Evaluating AI\u2019s Ability to Replicate AI Research. (April 2025)."},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3214239.3214242"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","unstructured":"Victoria Stodden Jennifer Seiler and Zhaokun Ma. 2018. An Empirical Analysis of Journal Policy Effectiveness for Computational Reproducibility. Proceedings of the National Academy of Sciences 115 11 (March 2018) 2584\u20132589. 10.1073\/pnas.1708290115","DOI":"10.1073\/pnas.1708290115"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","unstructured":"Ana Trisovic Matthew\u00a0K. Lau Thomas Pasquier and Merc\u00e8 Crosas. 2022. A Large-Scale Study on Research Code Quality and Execution. Scientific Data 9 1 (Feb. 2022) 60. 10.1038\/s41597-022-01143-6","DOI":"10.1038\/s41597-022-01143-6"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","unstructured":"Mark\u00a0D. Wilkinson Michel Dumontier IJsbrand\u00a0Jan Aalbersberg Gabrielle Appleton Myles Axton Arie Baak Niklas Blomberg Jan-Willem Boiten Luiz\u00a0Bonino da Silva Santos Philip\u00a0E. Bourne Jildau Bouwman Anthony\u00a0J. Brookes Tim Clark Merc\u00e8 Crosas Ingrid Dillo Olivier Dumon Scott Edmunds Chris\u00a0T. Evelo Richard Finkers Alejandra Gonzalez-Beltran Alasdair J.\u00a0G. Gray Paul Groth Carole Goble Jeffrey\u00a0S. Grethe Jaap Heringa Peter A.\u00a0C. \u2019t Hoen Rob Hooft Tobias Kuhn Ruben Kok Joost Kok Scott\u00a0J. Lusher Maryann\u00a0E. Martone Albert Mons Abel\u00a0L. Packer Bengt Persson Philippe Rocca-Serra Marco Roos Rene van Schaik Susanna-Assunta Sansone Erik Schultes Thierry Sengstag Ted Slater George Strawn Morris\u00a0A. Swertz Mark Thompson Johan van der Lei Erik van Mulligen Jan Velterop Andra Waagmeester Peter Wittenburg Katherine Wolstencroft Jun Zhao and Barend Mons. 2016. The FAIR Guiding Principles for Scientific Data Management and Stewardship. Scientific Data 3 1 (March 2016) 1\u20139. 10.1038\/sdata.2016.18","DOI":"10.1038\/sdata.2016.18"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","unstructured":"Joseph Wonsil Nichole Boufford Prakhar Agrawal Christopher Chen Tianhang Cui Akash Sivaram and Margo Seltzer. 2023. Reproducibility as a Service. Software: Practice and Experience 53 7 (2023) 1543\u20131571. 10.1002\/spe.3202","DOI":"10.1002\/spe.3202"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3589806.3600039"},{"key":"e_1_3_3_2_35_2","unstructured":"Kenichi Yasukata. 2024. Yasukata\/ELISA. https:\/\/github.com\/yasukata\/ELISA."},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3582016.3582042"}],"event":{"name":"ACM REP '25: ACM Conference on Reproducibility and Replicability","location":"Vancouver Canada","acronym":"ACM REP '25","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 3rd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3736731.3746157","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,9]],"date-time":"2026-01-09T18:04:18Z","timestamp":1767981858000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3736731.3746157"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,29]]},"references-count":35,"alternative-id":["10.1145\/3736731.3746157","10.1145\/3736731"],"URL":"https:\/\/doi.org\/10.1145\/3736731.3746157","relation":{},"subject":[],"published":{"date-parts":[[2025,7,29]]},"assertion":[{"value":"2025-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}