{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,25]],"date-time":"2026-08-25T02:13:40Z","timestamp":1787624020376,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":93,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663630","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"110-120","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["The Idealized Machine Learning Pipeline (IMLP) for Advancing Reproducibility in Machine Learning"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-7990-0426","authenticated-orcid":false,"given":"Yantong","family":"Zheng","sequence":"first","affiliation":[{"name":"University of Southern California, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2015-7825","authenticated-orcid":false,"given":"Victoria","family":"Stodden","sequence":"additional","affiliation":[{"name":"University of Southern California, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"US population: From","year":"1900","unstructured":"2000. US population: From 1900. http:\/\/demographia.com\/db-uspop1900.htm"},{"key":"e_1_3_2_1_2_1","unstructured":"2020. Artifact Review and Badging. https:\/\/www.acm.org\/publications\/policies\/artifact-review-and-badging-current"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jocm.2018.07.002"},{"key":"e_1_3_2_1_4_1","unstructured":"American\u00a0Psychological Association. 2022. Human Factors Psychology Studies Humans and Machines. https:\/\/www.apa.org\/education-career\/guide\/subfields\/human-factors."},{"key":"e_1_3_2_1_5_1","unstructured":"INTERNATIONAL\u00a0ERGONOMICS ASSOCIATION. 2003. IEA Triennial Report 2000-2003."},{"key":"e_1_3_2_1_6_1","volume-title":"Nature 533, 7604","author":"Baker Monya","year":"2016","unstructured":"Monya Baker. 2016. 1,500 scientists lift the lid on reproducibility. Nature 533, 7604 (2016)."},{"key":"e_1_3_2_1_7_1","first-page":"43","article-title":"Organizational research: Determining appropriate sample size in survey research","volume":"19","author":"Barlett E.","year":"2001","unstructured":"James\u00a0E. Barlett, Joe\u00a0W. Kotrlik, and Chadwick\u00a0C. Higgins. 2001. Organizational research: Determining appropriate sample size in survey research. Information Technology, Learning, and Performance Journal 19, 1 (2001), 43\u201350. Copyright - Copyright Organizational Systems Research Association Spring 2001; Last updated - 2022-10-20; SubjectsTermNotLitGenreText - United States\u2013US.","journal-title":"Information Technology, Learning, and Performance Journal"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.3389\/frai.2023.1130190"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/cns.2014.6997492"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1080\/15265161.2020.1819469"},{"key":"e_1_3_2_1_11_1","first-page":"786","article-title":"Incentives and careers in academia: theory and empirical analysis","volume":"103","author":"Checchi Daniele","year":"2021","unstructured":"Daniele Checchi, Gianni De\u00a0Fraja, and Stefano Verzillo. 2021. Incentives and careers in academia: theory and empirical analysis. Review of Economics and Statistics 103, 4 (2021), 786\u2013802.","journal-title":"Review of Economics and Statistics"},{"key":"e_1_3_2_1_12_1","unstructured":"CRediT. 2015. Contributor Roles Taxonomy. https:\/\/credit.niso.org\/."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1158\/1078-0432.ccr-07-0443"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1177\/00202940221083263"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jval.2019.02.012"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1198\/016214502753479248"},{"key":"e_1_3_2_1_17_1","volume-title":"Letter from the Editor","author":"Elston M","year":"2022","unstructured":"Dirk\u00a0M Elston. 2022. Letter from the Editor: Faulty generalization. Journal of the American Academy of Dermatology (2022)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/154193128703101219"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3312905"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12874-020-01055-2"},{"key":"e_1_3_2_1_21_1","volume-title":"Doi registrations for software. Front Matter","author":"Fenner Martin","year":"2018","unstructured":"Martin Fenner. 2018. Doi registrations for software. Front Matter (2018)."},{"key":"e_1_3_2_1_22_1","volume-title":"My life and work: In collaboration with Samuel Crowther","author":"Ford Henry","unstructured":"Henry Ford and Samuel Crowther. 1922. My life and work: In collaboration with Samuel Crowther. Cornstalk Publishing Company."},{"key":"e_1_3_2_1_23_1","volume-title":"Context: The Missing Piece in the Machine Learning Lifecycle. In KDD. https:\/\/rlnsanz.github.io\/dat\/Flor_CMI_18_CameraReady.pdf","author":"Garcia Rolando","year":"2018","unstructured":"Rolando Garcia, Vikram Sreekanti, Neeraja Yadwadkar, Daniel Crankshaw, Joseph\u00a0E Gonzalez, and Joseph\u00a0M Hellerstein. 2018. Context: The Missing Piece in the Machine Learning Lifecycle. In KDD. https:\/\/rlnsanz.github.io\/dat\/Flor_CMI_18_CameraReady.pdf"},{"key":"e_1_3_2_1_24_1","volume-title":"Human factors and usability for health information technology: old and new challenges. Yearbook of medical informatics 28, 01","author":"Gawande A","year":"2019","unstructured":"A Gawande, N Staggers, B Elias, E Makar, G Alexander, S Czaja, C Zarcadoolas, W Vaughon, C Lee, M Rockoff, 2019. Human factors and usability for health information technology: old and new challenges. Yearbook of medical informatics 28, 01 (2019), 071\u2013077."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.2202\/1544-6115.1034"},{"key":"e_1_3_2_1_26_1","volume-title":"Could machine learning fuel a reproducibility crisis in science?Nature news (Jul","author":"Gibney Elizabeth","year":"2022","unstructured":"Elizabeth Gibney. 2022. Could machine learning fuel a reproducibility crisis in science?Nature news (Jul 2022)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"Todd Golub. 2023. golubEsets: exprSets for golub leukemia data. https:\/\/doi.org\/10.18129\/B9.bioc.golubEsets R package version 1.42.0.","DOI":"10.18129\/B9.bioc.golubEsets"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.286.5439.531"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11503"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-020-09260-6"},{"key":"e_1_3_2_1_31_1","unstructured":"Steve Harrison Deborah Tatar and Phoebe Sengers. 2007. The three paradigms of HCI. In Alt. Chi. Session at the SIGCHI Conference on human factors in computing systems San Jose California USA. 1\u201318."},{"key":"e_1_3_2_1_32_1","volume-title":"International Conference on Innovative Techniques and Applications of Artificial Intelligence. Springer, 209\u2013222","author":"Howley Tom","year":"2005","unstructured":"Tom Howley, Michael\u00a0G Madden, Marie-Louise O\u2019Connell, and Alan\u00a0G Ryder. 2005. The effect of principal component analysis on machine learning accuracy with high dimensional spectral data. In International Conference on Innovative Techniques and Applications of Artificial Intelligence. Springer, 209\u2013222."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Matthew Hutson. 2018. Artificial intelligence faces reproducibility crisis.","DOI":"10.1126\/science.359.6377.725"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/eScience.2019.00017"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1913039116"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372829"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983924"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/17.12.1131"},{"key":"e_1_3_2_1_39_1","volume-title":"Dryad news. (Jul","author":"Lippincott Sarah","year":"2023","unstructured":"Sarah Lippincott. 2023. Dryad news. (Jul 2023)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.3846\/16487788.2016.1196559"},{"key":"e_1_3_2_1_41_1","unstructured":"Richard\u00a0L McKinley. 2000. Communication and localization with hearing protectors."},{"key":"e_1_3_2_1_42_1","volume-title":"Scientific integrity issues in environmental toxicology and chemistry: improving research reproducibility, credibility, and transparency. Integrated environmental assessment and management 15, 3","author":"Mebane A","year":"2019","unstructured":"Christopher\u00a0A Mebane, John\u00a0P Sumpter, Anne Fairbrother, Thomas\u00a0P Augspurger, Timothy\u00a0J Canfield, William\u00a0L Goodfellow, Patrick\u00a0D Guiney, Anne LeHuray, Lorraine Maltby, David\u00a0B Mayfield, 2019. Scientific integrity issues in environmental toxicology and chemistry: improving research reproducibility, credibility, and transparency. Integrated environmental assessment and management 15, 3 (2019), 320\u2013344."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3457607"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"crossref","unstructured":"Sid Mejdal Michael McCauley and Michael Beringer. 2001. Human Factors Design Guidelines for Multifunction Displays. Technical Report. Office of Aerospace Medicine.","DOI":"10.1037\/e403242004-001"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1080\/10803548.2006.11076669"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISTAS48451.2019.8937920"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2019.107077"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.5555\/3291291.3291297"},{"key":"e_1_3_2_1_50_1","unstructured":"Engineering National Academies\u00a0of Sciences Medicine 2019. Reproducibility and replicability in science. (2019)."},{"key":"e_1_3_2_1_51_1","unstructured":"National\u00a0Institutes of Health. 2000. Cancer Stat Facts: Leukemia. https:\/\/seer.cancer.gov\/statfacts\/html\/leuks.html Accessed: 2023-07-23."},{"key":"e_1_3_2_1_52_1","volume-title":"dagger: A python framework for reproducible machine learning experiment orchestration. arXiv preprint arXiv:2006.07484","author":"Paganini Michela","year":"2020","unstructured":"Michela Paganini and Jessica\u00a0Zosa Forde. 2020. dagger: A python framework for reproducible machine learning experiment orchestration. arXiv preprint arXiv:2006.07484 (2020)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4939-0742-7_8"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357160"},{"key":"e_1_3_2_1_55_1","unstructured":"Kayur Patel James Fogarty James\u00a0A Landay and Beverly\u00a0L Harrison. 2008. Examining Difficulties Software Developers Encounter in the Adoption of Statistical Machine Learning. (2008) 1563\u20131566."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1740-9713.2015.00827.x"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.5555\/3546258.3546422"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.3389\/fspas.2023.1134141"},{"key":"e_1_3_2_1_59_1","volume-title":"A taxonomy of tools for reproducible machine learning experiments. AIxIA 2021","author":"Quaranta Luigi","year":"2021","unstructured":"Luigi Quaranta, Fabio Calefato, and Filippo Lanubile. 2021. A taxonomy of tools for reproducible machine learning experiments. AIxIA 2021 (2021)."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.mex.2023.102364"},{"key":"e_1_3_2_1_61_1","volume-title":"Model evaluation, model selection, and algorithm selection in machine learning. arXiv preprint arXiv:1811.12808","author":"Raschka Sebastian","year":"2018","unstructured":"Sebastian Raschka. 2018. Model evaluation, model selection, and algorithm selection in machine learning. arXiv preprint arXiv:1811.12808 (2018)."},{"key":"e_1_3_2_1_62_1","volume-title":"rules, and knowledge","author":"Rasmussen Jens","year":"1983","unstructured":"Jens Rasmussen. 1983. Skills, rules, and knowledge; signals, signs, and symbols, and other distinctions in human performance models. IEEE transactions on systems, man, and cybernetics3 (1983), 257\u2013266."},{"key":"e_1_3_2_1_63_1","volume-title":"Human error","author":"Reason James","unstructured":"James Reason. 1990. Human error. Cambridge university press."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1111\/ijlh.13089"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341164"},{"key":"e_1_3_2_1_66_1","volume-title":"Handbook of human factors and ergonomics","author":"Salvendy Gavriel","unstructured":"Gavriel Salvendy and Waldemar Karwowski. 2021. Handbook of human factors and ergonomics. John Wiley & Sons."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.2006.12117"},{"key":"e_1_3_2_1_68_1","volume-title":"An introduction to human factors in medical devices","author":"Sawyer Dick","year":"1996","unstructured":"Dick Sawyer, KJ Aziz, CL Backinger, ET Beers, A Lowery, and SM Sykes. 1996. An introduction to human factors in medical devices. US Department of Health and Human Services, Public Health Service, Food and Drug Administration, Center for Devices and Radiological Health (1996), 55."},{"key":"e_1_3_2_1_69_1","volume-title":"An introduction to human factors in medical devices","author":"Sawyer Dick","year":"1996","unstructured":"Dick Sawyer, KJ Aziz, CL Backinger, ET Beers, A Lowery, and SM Sykes. 1996. An introduction to human factors in medical devices. US Department of Health and Human Services, Public Health Service, Food and Drug Administration, Center for Devices and Radiological Health (1996), 55."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.3390\/s22145205"},{"key":"e_1_3_2_1_71_1","first-page":"2347","article-title":"Conception of data preprocessing and partitioning procedure for machine learning algorithm","volume":"1","author":"Shrivastava Himanshu","year":"2013","unstructured":"Himanshu Shrivastava and Srivatsan Sridharan. 2013. Conception of data preprocessing and partitioning procedure for machine learning algorithm. International Journal of Recent Advances in Engineering & Technology (IJRAET) 1, 3 (2013), 2347\u20132812.","journal-title":"International Journal of Recent Advances in Engineering & Technology (IJRAET)"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2022.12.382"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.60087\/jklst.vol2.n3.p340"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/332306.332564"},{"key":"e_1_3_2_1_75_1","volume-title":"Human factors in manufacturing. Handbook of human factors and ergonomics","author":"Spath Dieter","year":"2012","unstructured":"Dieter Spath, Martin Braun, and Katrin Meinken. 2012. Human factors in manufacturing. Handbook of human factors and ergonomics (2012), 1643\u20131666."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"crossref","unstructured":"Ariel\u00a0Dora Stern. 2022. The Regulation of Medical AI: Policy Approaches Data and Innovation Incentives. Technical Report. National Bureau of Economic Research.","DOI":"10.3386\/w30639"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1109\/mcse.2009.19"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/3360646"},{"key":"e_1_3_2_1_79_1","volume-title":"Best practices for computational science: Software infrastructure and environments for reproducible and extensible research. Available at SSRN 2322276","author":"Stodden Victoria","year":"2013","unstructured":"Victoria Stodden and Sheila Miguez. 2013. Best practices for computational science: Software infrastructure and environments for reproducible and extensible research. Available at SSRN 2322276 (2013)."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSW.2018.8439914"},{"key":"e_1_3_2_1_81_1","volume-title":"Investigating human error: Incidents, accidents, and complex systems","author":"Strauch Barry","unstructured":"Barry Strauch. 2017. Investigating human error: Incidents, accidents, and complex systems. CRC Press."},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1810.04570"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1115\/IPC2004-0297"},{"key":"e_1_3_2_1_84_1","volume-title":"How pipelines make the oil market work\u2013Their networks, operation and regulation","author":"Trench J","year":"2001","unstructured":"Cheryl\u00a0J Trench. 2001. How pipelines make the oil market work\u2013Their networks, operation and regulation. Allegro Energy Group, New York (2001)."},{"key":"e_1_3_2_1_85_1","unstructured":"Valohai. 2020. What Is a Machine Learning Pipeline?https:\/\/valohai.com\/machine-learning-pipeline\/"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1002689"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1002\/cem.2446"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2005.07.007"},{"key":"e_1_3_2_1_89_1","volume-title":"Training data is more valuable than you think: A simple and effective method by retrieving from training data. arXiv preprint arXiv:2203.08773","author":"Wang Shuohang","year":"2022","unstructured":"Shuohang Wang, Yichong Xu, Yuwei Fang, Yang Liu, Siqi Sun, Ruochen Xu, Chenguang Zhu, and Michael Zeng. 2022. Training data is more valuable than you think: A simple and effective method by retrieving from training data. arXiv preprint arXiv:2203.08773 (2022)."},{"key":"e_1_3_2_1_90_1","volume-title":"its meaning and effect","author":"Whitehead Alfred\u00a0North","unstructured":"Alfred\u00a0North Whitehead. 1927. Symbolism, its meaning and effect. Macmillan."},{"key":"e_1_3_2_1_91_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2020.07.061"},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196709.3196729"},{"key":"e_1_3_2_1_93_1","unstructured":"Yantong Zheng. 2024. An Ideal Machine Learning Pipeline. https:\/\/github.com\/Machine-Learning-Pipelines\/IdealMachineLearningPipeline."}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","location":"Rennes France","acronym":"ACM REP '24","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663630","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663630","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:04:16Z","timestamp":1755914656000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663630"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":93,"alternative-id":["10.1145\/3641525.3663630","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663630","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}