{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T02:49:21Z","timestamp":1768013361800,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Portuguese Foundation for Science and Technology (FCT)","award":["SFRH\/BD\/151366\/2021"],"award-info":[{"award-number":["SFRH\/BD\/151366\/2021"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663623","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"46-51","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Evaluating Tools for Enhancing Reproducibility in Computational Scientific Experiments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6317-8792","authenticated-orcid":false,"given":"L\u00e1zaro","family":"Costa","sequence":"first","affiliation":[{"name":"INESC TEC, Portugal and University of Porto, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2198-3715","authenticated-orcid":false,"given":"Susana","family":"Barbosa","sequence":"additional","affiliation":[{"name":"INESC TEC, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4713-3834","authenticated-orcid":false,"given":"J\u00e1come","family":"Cunha","sequence":"additional","affiliation":[{"name":"INESC TEC, Portugal and University of Porto, Portugal"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/eScience55777.2022.00015"},{"key":"e_1_3_2_1_2_1","volume-title":"Thirty-fourth International Conference on Machine Learning.","author":"Anjos Andr\u00e9","year":"2017","unstructured":"Andr\u00e9 Anjos, Laurent El\u00a0Shafey, and S\u00e9bastien Marcel. 2017. Beat: An open-science web platform. In Thirty-fourth International Conference on Machine Learning."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12859-014-0369-z"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.envsoft.2015.12.010"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.22634"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2017.12.029"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-2544-7_5"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.00cd8f85"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.25080\/Majora-4af1f417-011"},{"key":"e_1_3_2_1_10_1","volume-title":"https:\/\/github.com\/mlcommons\/ck [Online","author":"Commons ML","year":"2024","unstructured":"ML Commons. 2024. Collective Mind (CM). https:\/\/github.com\/mlcommons\/ck [Online; accessed April-2024]."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.10651131"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587157"},{"key":"e_1_3_2_1_13_1","volume-title":"Incremental Query Execution. https:\/\/bitbucket.org\/TonHai\/iqe\/src\/master\/ [Online","author":"DBGroup","year":"2024","unstructured":"DBGroup. 2019. Incremental Query Execution. https:\/\/bitbucket.org\/TonHai\/iqe\/src\/master\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.3820"},{"key":"e_1_3_2_1_15_1","volume-title":"DVC - Data Version Control. https:\/\/dvc.org\/ [Online","author":"Data","year":"2024","unstructured":"Data Version\u00a0Control (DVC). 2024. DVC - Data Version Control. https:\/\/dvc.org\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.5598074"},{"key":"e_1_3_2_1_17_1","volume-title":"https:\/\/www.sciencedirect.com\/ [Online","year":"2024","unstructured":"Elsevier. 2024. ScienceDirect. https:\/\/www.sciencedirect.com\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_18_1","volume-title":"https:\/\/huggingface.co\/ [Online","author":"Face Hugging","year":"2024","unstructured":"Hugging Face. 2024. Hugging Face. https:\/\/huggingface.co\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_19_1","volume-title":"ACM Digital Library. https:\/\/dl.acm.org\/ [Online","author":"Association for Computing\u00a0Machinery. 2024.","year":"2024","unstructured":"Association for Computing\u00a0Machinery. 2024. ACM Digital Library. https:\/\/dl.acm.org\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2012.76"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1186\/gb-2010-11-8-r86"},{"key":"e_1_3_2_1_22_1","article-title":"The fundamental principles of reproducibility","volume":"379","author":"Gundersen Odd\u00a0Erik","year":"2021","unstructured":"Odd\u00a0Erik Gundersen. 2021. The fundamental principles of reproducibility. Philosophical Transactions of the Royal Society A 379, 2197 (2021), 20200210.","journal-title":"Philosophical Transactions of the Royal Society A"},{"key":"e_1_3_2_1_23_1","volume-title":"2011 USENIX Annual Technical Conference (USENIX ATC 11)","author":"J.","unstructured":"Philip\u00a0J. Guo and Dawson Engler. 2011. CDE: Using System Call Interposition to Automatically Create Portable Software Packages. In 2011 USENIX Annual Technical Conference (USENIX ATC 11). USENIX Association, Portland, OR. https:\/\/www.usenix.org\/conference\/usenixatc11\/cde-using-system-call-interposition-automatically-create-portable-software"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.2218\/ijdc.v10i1.370"},{"key":"e_1_3_2_1_25_1","volume-title":"https:\/\/ieeexplore.ieee.org\/Xplore\/home.jsp [Online","author":"Explore IEEE.","year":"2024","unstructured":"IEEE. 2024. IEEE Explore. https:\/\/ieeexplore.ieee.org\/Xplore\/home.jsp [Online; accessed February-2024]."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/eScience.2016.7870886"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3186266"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2618137.2618138"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPSW.2017.157"},{"key":"e_1_3_2_1_30_1","volume-title":"Your Machine Learning and Data Science Community. https:\/\/www.kaggle.com\/ [Online","year":"2024","unstructured":"Kaggle. 2024. Your Machine Learning and Data Science Community. https:\/\/www.kaggle.com\/ [Online; accessed February-2024],."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2002.1027796"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/2665008.2665035"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-39940-9_1471"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1002\/cpe.994"},{"key":"e_1_3_2_1_35_1","volume-title":"SOLE: towards descriptive and interactive publications. Implementing reproducible research 33","author":"Malik Tanu","year":"2014","unstructured":"Tanu Malik, Quan Pham, Ian\u00a0T Foster, F Leisch, and RD Peng. 2014. SOLE: towards descriptive and interactive publications. Implementing reproducible research 33 (2014)."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2755979.2755982"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3311790.3399619"},{"key":"e_1_3_2_1_38_1","volume-title":"Food Inspection Evaluation Predictions - Source Code. https:\/\/github.com\/Chicago\/food-inspections-evaluation [Online","author":"Chicago City","year":"2024","unstructured":"City of Chicago. 2016. Food Inspection Evaluation Predictions - Source Code. https:\/\/github.com\/Chicago\/food-inspections-evaluation [Online; accessed February-2024]."},{"key":"e_1_3_2_1_39_1","volume-title":"Information, Division on Engineering, Physical Sciences, Committee on Applied, Theoretical Statistics, Board on Mathematical\u00a0Sciences","author":"National","year":"2019","unstructured":"National\u00a0Academies of Sciences, Policy, Global Affairs, Board on Research\u00a0Data, Information, Division on Engineering, Physical Sciences, Committee on Applied, Theoretical Statistics, Board on Mathematical\u00a0Sciences, 2019. Reproducibility and replicability in science. National Academies Press."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.042781334"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.5555\/2482613.2482615"},{"key":"e_1_3_2_1_42_1","volume-title":"Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track.","author":"Ramakrishnan Chandrasekhar","year":"2023","unstructured":"Chandrasekhar Ramakrishnan, Michele Volpi, Fernando Perez-Cruz, Lilian Gasser, Firat Ozdemir, Patrick Paitz, Mohammad Alisafaee, Philipp Fischer, Ralf Grubenmann, Eliza\u00a0Jean Harris, 2023. Renku: a platform for sustainable data science. In Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track."},{"key":"e_1_3_2_1_43_1","volume-title":"Making Jupyter Notebooks Reproducible with ReproZip. https:\/\/docs.reprozip.org\/en\/1.x\/jupyter.html [Online","year":"2024","unstructured":"ReproZip. 2016. Making Jupyter Notebooks Reproducible with ReproZip. https:\/\/docs.reprozip.org\/en\/1.x\/jupyter.html [Online; accessed April-2024]."},{"key":"e_1_3_2_1_44_1","volume-title":"https:\/\/sciunit.run\/examples\/ [Online","author":"sciunit Group","year":"2024","unstructured":"sciunit Group. 2024. Sciunit. https:\/\/sciunit.run\/examples\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_45_1","volume-title":"https:\/\/link.springer.com\/ [Online","author":"Link Springer","year":"2024","unstructured":"Springer. 2024. Springer Link. https:\/\/link.springer.com\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/EDUCON.2016.7474573"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.29173\/iq969"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2015.18"},{"key":"e_1_3_2_1_49_1","first-page":"9","article-title":"Parrot: An application environment for data-intensive computing","volume":"6","author":"Thain Douglas","year":"2005","unstructured":"Douglas Thain and Miron Livny. 2005. Parrot: An application environment for data-intensive computing. Scalable Computing: Practice and Experience 6, 3 (2005), 9\u201318.","journal-title":"Scalable Computing: Practice and Experience"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/eScience.2017.51"},{"key":"e_1_3_2_1_51_1","volume-title":"Machine Learning and Knowledge Discovery in Databases, Hendrik Blockeel, Kristian Kersting, Siegfried Nijssen, and Filip \u017delezn\u00fd (Eds.)","author":"van Rijn N.","unstructured":"Jan\u00a0N. van Rijn, Bernd Bischl, Luis Torgo, Bo Gao, Venkatesh Umaashankar, Simon Fischer, Patrick Winter, Bernd Wiswedel, Michael\u00a0R. Berthold, and Joaquin Vanschoren. 2013. OpenML: A Collaborative Science Platform. In Machine Learning and Knowledge Discovery in Databases, Hendrik Blockeel, Kristian Kersting, Siegfried Nijssen, and Filip \u017delezn\u00fd (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 645\u2013649."},{"key":"e_1_3_2_1_52_1","volume-title":"Papers With Code. https:\/\/paperswithcode.com\/ [Online","author":"Code Papers","year":"2024","unstructured":"Papers with Code. 2024. Papers With Code. https:\/\/paperswithcode.com\/ [Online; accessed February-2024]."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/eScience.2019.00040"}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","location":"Rennes France","acronym":"ACM REP '24","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663623","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663623","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:05:30Z","timestamp":1755914730000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663623"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":53,"alternative-id":["10.1145\/3641525.3663623","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663623","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}