{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T01:45:23Z","timestamp":1787017523659,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3663529.3663778","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T15:43:13Z","timestamp":1720626193000},"page":"462-466","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Reproducibility Debt: Challenges and Future Pathways"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3416-2991","authenticated-orcid":false,"given":"Zara","family":"Hassan","sequence":"first","affiliation":[{"name":"Australian National University, Canberra, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6919-2149","authenticated-orcid":false,"given":"Christoph","family":"Treude","sequence":"additional","affiliation":[{"name":"Singapore Management University, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1163-8467","authenticated-orcid":false,"given":"Michael","family":"Norrish","sequence":"additional","affiliation":[{"name":"Australian National University, Canberra, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7041-4127","authenticated-orcid":false,"given":"Graham","family":"Williams","sequence":"additional","affiliation":[{"name":"Australian National University, Canberra, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4242-2725","authenticated-orcid":false,"given":"Alex","family":"Potanin","sequence":"additional","affiliation":[{"name":"Australian National University, Canberra, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/mcse.2021.3072860"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.10.008"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","unstructured":"Lorena A. Barba. 2018. Terminologies for Reproducible Research. https:\/\/doi.org\/10.48550\/ARXIV.1802.03311 10.48550\/ARXIV.1802.03311","DOI":"10.48550\/ARXIV.1802.03311"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2017.12.029"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/CANOPIEHPC51917.2020.00008"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2014.30"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/OCS.2012.30"},{"key":"e_1_3_2_1_9_1","volume-title":"Evaluation Methods for Machine Learning Workshop at the 26th ICML. 1, National Research Council of Canada Montreal, Canada.","author":"Drummond Chris","year":"2009","unstructured":"Chris Drummond. 2009. Replicability is not Reproducibility: Nor is it Good Science. In Evaluation Methods for Machine Learning Workshop at the 26th ICML. 1, National Research Council of Canada Montreal, Canada."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.envsoft.2017.01.024"},{"key":"e_1_3_2_1_11_1","unstructured":"FASEB. 2019. National Academies Releases Report on Reproducibility and Replicability in Science. https:\/\/www.faseb.org\/journals-and-news\/washington-update\/national-academies-releases-report-on-reproducibility-and-replicability-in-science"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.10.018"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-011-9181-9"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.07.011"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3186266"},{"key":"e_1_3_2_1_16_1","volume-title":"Philippe B Kruchten, Robert L. Nord, Will Snipes, and Paris Avgeriou.","author":"Izurieta Clemente","year":"2016","unstructured":"Clemente Izurieta, Ipek Ozkaya, Carolyn Budinger Seaman, Philippe B Kruchten, Robert L. Nord, Will Snipes, and Paris Avgeriou. 2016. Perspectives on Managing Technical Debt: A Transition Point and Roadmap from Dagstuhl. In QuASoQ\/TDA@APSEC."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/INFCOMW.2017.8116418"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.021651343"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2014.05.006"},{"key":"e_1_3_2_1_20_1","volume-title":"Guidelines for performing Systematic Literature Reviews in Software Engineering","author":"Kitchenham Barbara Ann","unstructured":"Barbara Ann Kitchenham and Stuart Charters. 2007. Guidelines for performing Systematic Literature Reviews in Software Engineering. Keele University and Durham University Joint Report."},{"key":"e_1_3_2_1_21_1","volume-title":"Dietmar Winkler","author":"Lenarduzzi Valentina","unstructured":"Valentina Lenarduzzi, Francesco Lomio, Sergio Moreschini, Davide Taibi, and Damian Andrew Tamburri. 2021. Software Quality for AI: Where We Are Now? In Software Quality: Future Perspectives on Software Engineering Quality, Dietmar Winkler, Stefan Biffl, Daniel Mendez, Manuel Wimmer, and Johannes Bergsmann (Eds.). Springer, Cham. 43\u201353."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10816-015-9272-9"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jocs.2015.04.012"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"Suchita Mukherjee Abigail Almanza and Cindy Rubio-Gonz\u00e1lez. 2021. Fixing dependency errors for Python build reproducibility. https:\/\/doi.org\/10.1145\/3460319.3464797 10.1145\/3460319.3464797","DOI":"10.1145\/3460319.3464797"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.17226\/25303"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.1213847"},{"key":"e_1_3_2_1_27_1","volume-title":"An External Replication. In IEEE 25th International Conference on Software Analysis, Evolution and Reengineering. IEEE","author":"Pinto G.","unstructured":"G. Pinto, I. Wiese, and L. F. Dias. 2018. How Do Scientists Develop Scientific Software? An External Replication. In IEEE 25th International Conference on Software Analysis, Evolution and Reengineering. IEEE, Campobasso, Italy. 582\u2013591."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2204.04372"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.05.010"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-44429-7_4"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3131151.3131164"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems -","volume":"2","author":"Sculley D.","year":"2015","unstructured":"D. Sculley, Gary Holt, Daniel Golovin, Eugene Davydov, Todd Phillips, Dietmar Ebner, Vinay Chaudhary, Michael Young, Jean-Francois Crespo, and Dan Dennison. 2015. Hidden technical debt in Machine learning systems. In Proceedings of the 28th International Conference on Neural Information Processing Systems - Volume 2 (NIPS\u201915). MIT Press, Cambridge, MA, USA. 2503\u20132511."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","unstructured":"Victoria Stodden. 2010. The Scientific Method in Practice: Reproducibility in the Computational Sciences. SSRN Electronic Journal 4773\u201310. issn:1556-5068 https:\/\/doi.org\/10.2139\/ssrn.1550193 10.2139\/ssrn.1550193","DOI":"10.2139\/ssrn.1550193"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2012.12.052"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00136"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR52588.2021.00030"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377812.3390803"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2601248.2601268"}],"event":{"name":"FSE '24: 32nd ACM International Conference on the Foundations of Software Engineering","location":"Porto de Galinhas Brazil","acronym":"FSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663778","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3663529.3663778","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:44:20Z","timestamp":1750275860000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663778"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":39,"alternative-id":["10.1145\/3663529.3663778","10.1145\/3663529"],"URL":"https:\/\/doi.org\/10.1145\/3663529.3663778","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}