{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,24]],"date-time":"2026-02-24T03:51:37Z","timestamp":1771905097829,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":73,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Laboratory Directed Research and Development","doi-asserted-by":"publisher","award":["229304"],"award-info":[{"award-number":["229304"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663624","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"52-63","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Towards Evidence-Based Software Quality Practices for Reproducibility: Practices and Aligned Software Qualities"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1743-3684","authenticated-orcid":false,"given":"Christian","family":"Gilbertson","sequence":"first","affiliation":[{"name":"Sandia National Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5283-2138","authenticated-orcid":false,"given":"Miranda","family":"Mundt","sequence":"additional","affiliation":[{"name":"Sandia National Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7767-0067","authenticated-orcid":false,"given":"Joshua","family":"Teves","sequence":"additional","affiliation":[{"name":"Sandia National Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7156-3885","authenticated-orcid":false,"given":"Simone","family":"Toribio","sequence":"additional","affiliation":[{"name":"Sandia National Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1701-0008","authenticated-orcid":false,"given":"Reed","family":"Milewicz","sequence":"additional","affiliation":[{"name":"Sandia National Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2020. Artifact Review and Badging Version 1.1. https:\/\/www.acm.org\/publications\/policies\/artifact-review-and-badging-current\/"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.2882355"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-58640-3_13"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.2881905"},{"key":"e_1_3_2_1_5_1","volume-title":"Digital Research Conference","author":"Baxter Rob","year":"2012","unstructured":"Rob Baxter, N\u00a0Chue Hong, Dirk Gorissen, James Hetherington, and Ilian Todorov. 2012. The research software engineer. In Digital Research Conference, Oxford. 1\u20133."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2021.3052101"},{"key":"e_1_3_2_1_7_1","volume-title":"Metamorphic runtime checking of applications without test oracles. CrossTalk 28, 2","author":"Bell Jonathan","year":"2015","unstructured":"Jonathan Bell, Christian Murphy, and Gail Kaiser. 2015. Metamorphic runtime checking of applications without test oracles. CrossTalk 28, 2 (2015)."},{"key":"e_1_3_2_1_8_1","volume-title":"Snowball sampling: Problems and techniques of chain referral sampling. Sociological methods & research 10, 2","author":"Biernacki Patrick","year":"1981","unstructured":"Patrick Biernacki and Dan Waldorf. 1981. Snowball sampling: Problems and techniques of chain referral sampling. Sociological methods & research 10, 2 (1981), 141\u2013163."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723882"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.5334\/jors.bu"},{"key":"e_1_3_2_1_11_1","volume-title":"Teoria statistica delle classi e calcolo delle probabilita. Pubblicazioni del R Istituto Superiore di Scienze Economiche e Commericiali di Firenze 8","author":"Bonferroni Carlo","year":"1936","unstructured":"Carlo Bonferroni. 1936. Teoria statistica delle classi e calcolo delle probabilita. Pubblicazioni del R Istituto Superiore di Scienze Economiche e Commericiali di Firenze 8 (1936), 3\u201362."},{"key":"e_1_3_2_1_12_1","volume-title":"Promoting transparency and reproducibility in enhanced molecular simulations. Nature methods 16, 8","author":"Bonomi Massimiliano","year":"2019","unstructured":"Massimiliano Bonomi. 2019. Promoting transparency and reproducibility in enhanced molecular simulations. Nature methods 16, 8 (2019), 670\u2013673."},{"key":"e_1_3_2_1_13_1","volume-title":"Research software engineers: state of the nation report","author":"Brett Alys","year":"2017","unstructured":"Alys Brett, Michael Croucher, Robert Haines, Simon Hettrick, James Hetherington, Mark Stillwell, and Claire Wyatt. 2017. Research software engineers: state of the nation report 2017. (2017)."},{"key":"e_1_3_2_1_14_1","unstructured":"Coral Calero Maria\u00a0\u00c1ngeles Moraga Manuel\u00a0F Bertoa and Leticia Duboc. 2014. Quality in Use and Software Greenability.. In RE4SuSy@ RE. 28\u201336."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-75857-2_3"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3322790.3330594"},{"key":"e_1_3_2_1_17_1","volume-title":"Toward Community-Driven big Open brain science: open big data and tools for structure, function, and genetics. Annual review of neuroscience 43","author":"Charles S","year":"2020","unstructured":"Adam\u00a0S Charles, Benjamin Falk, Nicholas Turner, Talmo\u00a0D Pereira, Daniel Tward, Benjamin\u00a0D Pedigo, Jaewon Chung, Randal Burns, Satrajit\u00a0S Ghosh, Justus\u00a0M Kebschull, 2020. Toward Community-Driven big Open brain science: open big data and tools for structure, function, and genetics. Annual review of neuroscience 43 (2020), 441\u2013464."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2020.2973362"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2017.12.005"},{"key":"e_1_3_2_1_20_1","volume-title":"Nextflow enables reproducible computational workflows. Nature biotechnology 35, 4","author":"Di\u00a0Tommaso Paolo","year":"2017","unstructured":"Paolo Di\u00a0Tommaso, Maria Chatzou, Evan\u00a0W Floden, Pablo\u00a0Prieto Barja, Emilio Palumbo, and Cedric Notredame. 2017. Nextflow enables reproducible computational workflows. Nature biotechnology 35, 4 (2017), 316\u2013319."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s42979-020-0077-x"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00355-011-0603-9"},{"key":"e_1_3_2_1_23_1","volume-title":"The vital importance of high-performance computing to US competitiveness","author":"Ezell J","year":"2016","unstructured":"Stephen\u00a0J Ezell and Robert\u00a0D Atkinson. 2016. The vital importance of high-performance computing to US competitiveness. Information Technology and Innovation Foundation 28 (April 2016)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213908"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/VS-GAMES.2015.7295790"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2881041"},{"key":"e_1_3_2_1_27_1","unstructured":"BG Glaser and AL Strauss. 1967. The discovery of grounded theory: strategies for qualitative research. (1967)."},{"key":"e_1_3_2_1_28_1","volume-title":"Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences. Genome biology 11, 8","author":"Goecks Jeremy","year":"2010","unstructured":"Jeremy Goecks, Anton Nekrutenko, and James Taylor. 2010. Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences. Genome biology 11, 8 (2010), 1\u201313."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2379636.2379651"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3589806.3600037"},{"key":"e_1_3_2_1_31_1","article-title":"The fundamental principles of reproducibility","volume":"379","author":"Gundersen Odd\u00a0Erik","year":"2021","unstructured":"Odd\u00a0Erik Gundersen. 2021. The fundamental principles of reproducibility. Philosophical Transactions of the Royal Society A 379, 2197 (2021), 20200210.","journal-title":"Philosophical Transactions of the Royal Society A"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11082-021-03192-4"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2020.2998235"},{"key":"e_1_3_2_1_34_1","volume-title":"Tools and Techniques for High Performance Computing: Selected Workshops, HUST, SE-HER and WIHPC, Held in Conjunction with SC","author":"Heroux A","year":"2019","unstructured":"Michael\u00a0A Heroux, Elsa Gonsiorowski, Rinku Gupta, Reed Milewicz, J\u00a0David Moulton, Gregory\u00a0R Watson, Jim Willenbring, Richard\u00a0J Zamora, and Elaine\u00a0M Raybourn. 2020. Lightweight software process improvement using productivity and sustainability improvement planning (PSIP). In Tools and Techniques for High Performance Computing: Selected Workshops, HUST, SE-HER and WIHPC, Held in Conjunction with SC 2019, Denver, CO, USA, November 17\u201318, 2019, Revised Selected Papers 6. Springer, 98\u2013110."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13194-021-00362-9"},{"key":"e_1_3_2_1_36_1","unstructured":"Sawtooth\u00a0Software Inc.2020. The MaxDiff System Technical Paper. https:\/\/sawtoothsoftware.com\/resources\/technical-papers\/maxdiff-technical-paper"},{"key":"e_1_3_2_1_37_1","volume-title":"The case for open computer programs. Nature 482, 7386","author":"Ince C","year":"2012","unstructured":"Darrel\u00a0C Ince, Leslie Hatton, and John Graham-Cumming. 2012. The case for open computer programs. Nature 482, 7386 (2012), 485\u2013488."},{"key":"e_1_3_2_1_38_1","unstructured":"ISO\/IEC. 2011. Systems and software engineering \u2014 Systems and software Quality Requirements and Evaluation (SQuaRE) \u2014 System and software quality models (25010:2011). https:\/\/www.iso.org\/standard\/35733.html."},{"key":"e_1_3_2_1_39_1","volume-title":"A review of key Likert scale development advances: 1995\u20132019. Frontiers in psychology 12","author":"Jebb T","year":"2021","unstructured":"Andrew\u00a0T Jebb, Vincent Ng, and Louis Tay. 2021. A review of key Likert scale development advances: 1995\u20132019. Frontiers in psychology 12 (2021), 637547."},{"key":"e_1_3_2_1_40_1","volume-title":"Four simple recommendations to encourage best practices in research software. F1000Research 6","author":"Jim\u00e9nez C","year":"2017","unstructured":"Rafael\u00a0C Jim\u00e9nez, Mateusz Kuzak, Monther Alhamdoosh, Michelle Barker, B\u00e9r\u00e9nice Batut, Mikael Borg, Salvador Capella-Gutierrez, Neil\u00a0Chue Hong, Martin Cook, Manuel Corpas, 2017. Four simple recommendations to encourage best practices in research software. F1000Research 6 (2017)."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.2883051"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.03221930"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.2883326"},{"key":"e_1_3_2_1_44_1","volume-title":"A software chasm: Software engineering and scientific computing","author":"Kelly F","year":"2007","unstructured":"Diane\u00a0F Kelly. 2007. A software chasm: Software engineering and scientific computing. IEEE software 24, 6 (2007), 120\u2013119."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/bts480"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1142\/S0218194018500146"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW.2019.00035"},{"key":"e_1_3_2_1_48_1","unstructured":"Benjamin\u00a0D Lee. 2018. Ten simple rules for documenting scientific software."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2012.38"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-45442-5_6"},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the fifth Berkeley symposium on mathematical statistics and probability, Vol.\u00a01","author":"MacQueen James","year":"1967","unstructured":"James MacQueen 1967. Some methods for classification and analysis of multivariate observations. In Proceedings of the fifth Berkeley symposium on mathematical statistics and probability, Vol.\u00a01. Oakland, CA, USA, 281\u2013297."},{"key":"e_1_3_2_1_52_1","first-page":"37","article-title":"Optimized software quality assurance model for testing scientific software","volume":"36","author":"Maheswari G\u00a0Uma","year":"2011","unstructured":"G\u00a0Uma Maheswari and VV\u00a0Rama Prasad. 2011. Optimized software quality assurance model for testing scientific software. International Journal of Computer Applications 36, 7 (2011), 37\u201346.","journal-title":"International Journal of Computer Applications"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3589806.3600040"},{"key":"e_1_3_2_1_54_1","unstructured":"Reed Milewicz Miranda Mundt Christian Gilbertson and Joshua Teves. 2023. Evidence-Based Software Quality Practices for Computational Reproducibility. Technical Report SAND2023-10506R."},{"key":"e_1_3_2_1_55_1","volume-title":"Working in Harmony: Towards Integrating RSEs into Multi-Disciplinary CSE Teams. arXiv preprint arXiv:2201.04010","author":"Mundt Miranda","year":"2022","unstructured":"Miranda Mundt and Reed Milewicz. 2022. Working in Harmony: Towards Integrating RSEs into Multi-Disciplinary CSE Teams. arXiv preprint arXiv:2201.04010 (2022)."},{"key":"e_1_3_2_1_56_1","volume-title":"Engineering, and Medicine","author":"National Academies of Sciences","year":"2019","unstructured":"National Academies of Sciences, Engineering, and Medicine 2019. Reproducibility and replicability in science. National Academies Press."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"crossref","unstructured":"Daniel N\u00fcst Vanessa Sochat Ben Marwick Stephen\u00a0J Eglen Tim Head Tony Hirst and Benjamin\u00a0D Evans. 2020. Ten simple rules for writing Dockerfiles for reproducible data science. e1008316\u00a0pages.","DOI":"10.31219\/osf.io\/fsd7t"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/505532.505535"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2019.00077"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-021-09961-9"},{"key":"e_1_3_2_1_61_1","volume-title":"Proceedings of the Workshop on Sustainable Software for Science: Practice and Experiences (WSSSPE5. 1). WSSSPE.","author":"Queiroz F","year":"2017","unstructured":"F Queiroz, R Silva, J Miller, S Brockhauser, and H Fangohr. 2017. Good Usability Practices in Scientific Software Development. In Proceedings of the Workshop on Sustainable Software for Science: Practice and Experiences (WSSSPE5. 1). WSSSPE."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.2172\/1845193"},{"key":"e_1_3_2_1_63_1","volume-title":"An integrated approach to communication theory and research","author":"Rogers M","unstructured":"Everett\u00a0M Rogers, Arvind Singhal, and Margaret\u00a0M Quinlan. 2014. Diffusion of innovations. In An integrated approach to communication theory and research. Routledge, 432\u2013448."},{"key":"e_1_3_2_1_64_1","volume-title":"Case study research in software engineering: Guidelines and examples","author":"Runeson Per","unstructured":"Per Runeson, Martin Host, Austen Rainer, and Bjorn Regnell. 2012. Case study research in software engineering: Guidelines and examples. John Wiley & Sons."},{"key":"e_1_3_2_1_65_1","volume-title":"Ten simple rules for reproducible computational research. PLoS computational biology 9, 10","author":"Sandve Geir\u00a0Kjetil","year":"2013","unstructured":"Geir\u00a0Kjetil Sandve, Anton Nekrutenko, James Taylor, and Eivind Hovig. 2013. Ten simple rules for reproducible computational research. PLoS computational biology 9, 10 (2013), e1003285."},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-014-9246-2"},{"key":"e_1_3_2_1_67_1","volume-title":"2012 IEEE 8th International Conference on E-Science. IEEE, 1\u20138.","author":"Stodden Victoria","year":"2012","unstructured":"Victoria Stodden, Christophe Hurlin, and Christophe P\u00e9rignon. 2012. RunMyCode. org: A novel dissemination and collaboration platform for executing published computational results. In 2012 IEEE 8th International Conference on E-Science. IEEE, 1\u20138."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2015.18"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1111\/1467-9868.00293"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-022-01143-6"},{"key":"e_1_3_2_1_71_1","volume-title":"Reproducible and reusable research: are journal data sharing policies meeting the mark?PeerJ 5","author":"Vasilevsky A","year":"2017","unstructured":"Nicole\u00a0A Vasilevsky, Jessica Minnier, Melissa\u00a0A Haendel, and Robin\u00a0E Champieux. 2017. Reproducible and reusable research: are journal data sharing policies meeting the mark?PeerJ 5 (2017), e3208."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"crossref","unstructured":"Craig Willis and Victoria Stodden. 2020. Trust but verify: How to leverage policies workflows and infrastructure to ensure computational reproducibility in publication. (2020).","DOI":"10.1162\/99608f92.25982dcf"},{"key":"e_1_3_2_1_73_1","volume-title":"International Workshop on Software Measurement and the 15th International Conference on Software Process and Product Measurement, Mexico.","author":"Y\u0131lmaz Nebi","year":"2020","unstructured":"Nebi Y\u0131lmaz and Ay\u00e7a\u00a0Koluk\u0131sa Tarhan. 2020. Meta-models for software quality and its evaluation: a systematic literature review. In International Workshop on Software Measurement and the 15th International Conference on Software Process and Product Measurement, Mexico."}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","location":"Rennes France","acronym":"ACM REP '24","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663624","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663624","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:05:37Z","timestamp":1755914737000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663624"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":73,"alternative-id":["10.1145\/3641525.3663624","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663624","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}