{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,18]],"date-time":"2026-07-18T02:36:27Z","timestamp":1784342187955,"version":"3.55.0"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T00:00:00Z","timestamp":1741564800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T00:00:00Z","timestamp":1741564800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,5]]},"DOI":"10.1007\/s10664-025-10620-6","type":"journal-article","created":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T01:51:26Z","timestamp":1741571486000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":16,"title":["Testing research software: an in-depth survey of practices, methods, and tools"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5228-4664","authenticated-orcid":false,"given":"Nasir U.","family":"Eisty","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5954-1616","authenticated-orcid":false,"given":"Upulee","family":"Kanewala","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7824-9151","authenticated-orcid":false,"given":"Jeffrey C.","family":"Carver","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,3,10]]},"reference":[{"issue":"4","key":"10620_CR1","first-page":"315","volume":"16","author":"BA Allan","year":"2008","unstructured":"Allan BA, Norris B, Elwasif WR, Armstrong RC (2008) Managing scientific software complexity with bocca and cca. Sci Program 16(4):315\u2013327","journal-title":"Sci Program"},{"key":"10620_CR2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110848","volume":"172","author":"EM Arvanitou","year":"2021","unstructured":"Arvanitou EM, Ampatzoglou A, Chatzigeorgiou A, Carver JC (2021) Software engineering practices for scientific software development: a systematic mapping study. J Syst Softw 172:110848","journal-title":"J Syst Softw"},{"key":"10620_CR3","unstructured":"Capgemini (2021) World quality report 2021-2022. Tech. rep., Capgemini, Sogeti, and Micro Focus. https:\/\/www.capgemini.com\/insights\/research-library\/world-quality-report-wqr-2021-22\/"},{"key":"10620_CR4","doi-asserted-by":"publisher","unstructured":"Chen TY, Kuo FC, Liu H, Poon PL, Towey D, Tse TH, Zhou ZQ (2018) Metamorphic testing: a review of challenges and opportunities. ACM Comput Surv 51(1). https:\/\/doi.org\/10.1145\/3143561","DOI":"10.1145\/3143561"},{"issue":"6","key":"10620_CR5","doi-asserted-by":"publisher","first-page":"49","DOI":"10.1109\/MS.2011.117","volume":"28","author":"T Clune","year":"2011","unstructured":"Clune T, Rood R (2011) Software testing and verification in climate model development. IEEE Softw 28(6):49\u201355. https:\/\/doi.org\/10.1109\/MS.2011.117","journal-title":"IEEE Softw"},{"key":"10620_CR6","doi-asserted-by":"publisher","unstructured":"Ding J, Zhang D, Hu XH (2016) An application of metamorphic testing for testing scientific software. Association for Computing Machinery, New York, NY, USA. https:\/\/doi.org\/10.1145\/2896971.2896981","DOI":"10.1145\/2896971.2896981"},{"key":"10620_CR7","doi-asserted-by":"publisher","unstructured":"Easterbrook SM (2010) Climate change: a grand software challenge. In: Proceedings of the FSE\/SDP workshop on future of software engineering research. Association for Computing Machinery, New York, NY, USA, FoSER \u201910, pp 99\u2013104. https:\/\/doi.org\/10.1145\/1882362.1882383","DOI":"10.1145\/1882362.1882383"},{"issue":"6","key":"10620_CR8","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1109\/MCSE.2009.193","volume":"11","author":"SM Easterbrook","year":"2009","unstructured":"Easterbrook SM, Johns TC (2009) Engineering the software for understanding climate change. Comput Sci Eng 11(6):65\u201374. https:\/\/doi.org\/10.1109\/MCSE.2009.193","journal-title":"Comput Sci Eng"},{"key":"10620_CR9","doi-asserted-by":"publisher","unstructured":"Eisty NU, Carver JC (2022) Testing research software: a survey. Empir Softw Eng 27(138). https:\/\/doi.org\/10.1007\/s10664-022-10184-9","DOI":"10.1007\/s10664-022-10184-9"},{"key":"10620_CR10","doi-asserted-by":"publisher","unstructured":"Eisty NU, Thiruvathukal GK, Carver JC (2018) A survey of software metric use in research software development. In: 2018 IEEE 14th international conference on e-Science (e-Science), pp 212\u2013222. https:\/\/doi.org\/10.1109\/eScience.2018.00036","DOI":"10.1109\/eScience.2018.00036"},{"key":"10620_CR11","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1007\/978-3-030-50436-6_33","volume-title":"Computational Science - ICCS 2020","author":"NU Eisty","year":"2020","unstructured":"Eisty NU, Perez D, Carver JC, Moulton JD, Nam HA (2020) Testing research software: a case study. In: Krzhizhanovskaya VV, Z\u00e1vodszky G, Lees MH, Dongarra JJ, Sloot PMA, Brissos S, Teixeira J (eds) Computational Science - ICCS 2020. Springer International Publishing, Cham, pp 457\u2013463"},{"key":"10620_CR12","unstructured":"EPA (2023) Storm water management model (swmm). https:\/\/www.epa.gov\/water-research\/storm-water-management-model-swmm"},{"key":"10620_CR13","doi-asserted-by":"crossref","unstructured":"Feitosa D, von Flach C, Costa J (2023) Understanding practices and challenges of developing sustainable research software: a pilot interview. In: Workshop de pr\u00e1ticas de ci\u00eancia aberta para engenharia de software (opensciense)","DOI":"10.5753\/opensciense.2023.235677"},{"key":"10620_CR14","doi-asserted-by":"publisher","unstructured":"Hannay JE, MacLeod C, Singer J, Langtangen HP, Pfahl D, Wilson G (2009) How do scientists develop and use scientific software? In: 2009 ICSE workshop on software engineering for computational science and engineering, pp 1\u20138. https:\/\/doi.org\/10.1109\/SECSE.2009.5069155","DOI":"10.1109\/SECSE.2009.5069155"},{"key":"10620_CR15","doi-asserted-by":"publisher","unstructured":"Heaton D, Carver JC (2015) Claims about the use of software engineering practices in science: a systematic literature review. Inf Softw Technol 67:207\u2013219. https:\/\/doi.org\/10.1016\/j.infsof.2015.07.011, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0950584915001342","DOI":"10.1016\/j.infsof.2015.07.011"},{"issue":"6","key":"10620_CR16","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1109\/MCSE.2023.3260475","volume":"24","author":"MA Heroux","year":"2022","unstructured":"Heroux MA (2022) Research software science: expanding the impact of research software engineering. Comput Sci Eng 24(6):22\u201327. https:\/\/doi.org\/10.1109\/MCSE.2023.3260475","journal-title":"Comput Sci Eng"},{"key":"10620_CR17","doi-asserted-by":"publisher","unstructured":"Hynninen T, Kasurinen J, Knutas A, Taipale O (2018) Software testing: survey of the industry practices. In: 2018 41st International convention on information and communication technology, electronics and microelectronics (MIPRO), pp 1449\u20131454. https:\/\/doi.org\/10.23919\/MIPRO.2018.8400261","DOI":"10.23919\/MIPRO.2018.8400261"},{"key":"10620_CR18","unstructured":"ITSQB (2018) ISTQB\u00ae Worldwide software testing practices report 2017-18. https:\/\/istqb-main-web-prod.s3.amazonaws.com\/media\/documents\/ISTQB_2017-18_Revised.pdf. Accessed 14 Aug 2024"},{"issue":"1","key":"10620_CR19","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1109\/MCSE.2018.2875368","volume":"21","author":"U Kanewala","year":"2019","unstructured":"Kanewala U, Yueh Chen T (2019) Metamorphic testing: a simple yet effective approach for testing scientific software. Comput Sci Eng 21(1):66\u201372. https:\/\/doi.org\/10.1109\/MCSE.2018.2875368","journal-title":"Comput Sci Eng"},{"key":"10620_CR20","doi-asserted-by":"publisher","unstructured":"Kanewala U, Bieman JM (2014) Testing scientific software: a systematic literature review. Inf Softw Technol 56(10):1219\u20131232. https:\/\/doi.org\/10.1016\/j.infsof.2014.05.006, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0950584914001232","DOI":"10.1016\/j.infsof.2014.05.006"},{"issue":"2","key":"10620_CR21","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1109\/MCSE.2018.2883051","volume":"21","author":"DS Katz","year":"2019","unstructured":"Katz DS, McInnes LC, Bernholdt DE, Mayes AC, Hong NPC, Duckles J, Gesing S, Heroux MA, Hettrick S, Jimenez RC, Pierce M, Weaver B, Wilkins-Diehr N (2019) Community organizations: changing the culture in which research software is developed and sustained. Comput Sci Eng 21(2):8\u201324. https:\/\/doi.org\/10.1109\/MCSE.2018.2883051","journal-title":"Comput Sci Eng"},{"key":"10620_CR22","unstructured":"Kelly D, S R, Saint R, Floor P, Sanders R, Kelly D (2008) The challenge of testing scientific software. In: Proceeding of conference for the association for software testing, pp 30\u201336"},{"issue":"7","key":"10620_CR23","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1009149","volume":"17","author":"CC Kerr","year":"2021","unstructured":"Kerr CC, Stuart RM, Mistry D, Abeysuriya RG, Rosenfeld K, Hart GR, N\u00fa\u00f1ez RC, Cohen JA, Selvaraj P, Hagedorn B et al (2021) Covasim: an agent-based model of covid-19 dynamics and interventions. PLoS Comput Biol 17(7):e1009149","journal-title":"PLoS Comput Biol"},{"key":"10620_CR24","doi-asserted-by":"publisher","unstructured":"Kochhar PS, Xia X, Lo D (2019) Practitioners\u2019 views on good software testing practices. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP), pp 61\u201370. https:\/\/doi.org\/10.1109\/ICSE-SEIP.2019.00015","DOI":"10.1109\/ICSE-SEIP.2019.00015"},{"key":"10620_CR25","doi-asserted-by":"publisher","unstructured":"Lukasczyk S, Fraser G (2022) Pynguin: automated unit test generation for python. In: Proceedings of the ACM\/IEEE 44th international conference on software engineering: companion proceedings. Association for Computing Machinery, New York, NY, USA, ICSE \u201922, pp 168\u2013172. https:\/\/doi.org\/10.1145\/3510454.3516829","DOI":"10.1145\/3510454.3516829"},{"issue":"11","key":"10620_CR26","doi-asserted-by":"publisher","first-page":"2312","DOI":"10.1109\/TSE.2019.2946563","volume":"47","author":"VJ Man\u00e8s","year":"2021","unstructured":"Man\u00e8s VJ, Han H, Han C, Cha SK, Egele M, Schwartz EJ, Woo M (2021) The art, science, and engineering of fuzzing: a survey. IEEE Trans Software Eng 47(11):2312\u20132331. https:\/\/doi.org\/10.1109\/TSE.2019.2946563","journal-title":"IEEE Trans Software Eng"},{"key":"10620_CR27","doi-asserted-by":"publisher","unstructured":"Miller G (2006) A scientist\u2019s nightmare: software problem leads to five retractions. Science 314(5807):1856\u20131857. https:\/\/doi.org\/10.1126\/science.314.5807.1856, https:\/\/science.sciencemag.org\/content\/314\/5807\/1856, https:\/\/science.sciencemag.org\/content\/314\/5807\/1856.full.pdf","DOI":"10.1126\/science.314.5807.1856"},{"issue":"6","key":"10620_CR28","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1109\/MCSE.2023.3256759","volume":"24","author":"MR Mundt","year":"2022","unstructured":"Mundt MR, Beattie K, Bisila J, Ferenbaugh CR, Godoy WF, Gupta R, Guyer JE, Kiran M, Malviya-Thakur A, Milewicz R, Sims BH, Sochat V, Teves JB (2022) For the public good: connecting, retaining, and recognizing current and future rses at u.s. national research laboratories and agencies. Comp Sci Eng 24(6):6\u201313. https:\/\/doi.org\/10.1109\/MCSE.2023.3256759","journal-title":"Comp Sci Eng"},{"key":"10620_CR29","doi-asserted-by":"publisher","unstructured":"Nie C, Leung H (2011) A survey of combinatorial testing. ACM Comput Surv 43(2). https:\/\/doi.org\/10.1145\/1883612.1883618","DOI":"10.1145\/1883612.1883618"},{"issue":"4","key":"10620_CR30","doi-asserted-by":"publisher","first-page":"399","DOI":"10.1177\/1094342004048534","volume":"18","author":"DE Post","year":"2004","unstructured":"Post DE, Kendall RP (2004) Software project management and quality engineering practices for complex, coupled multiphysics, massively parallel computational simulations: lessons learned from asci. Int J High Performance Comput Appl 18(4):399\u2013416. https:\/\/doi.org\/10.1177\/1094342004048534","journal-title":"Int J High Performance Comput Appl"},{"key":"10620_CR31","unstructured":"Sanders R, Kelly D (2008) The challenge of testing scientific software. In: Proceedings Conference for the Association for Software Testing (CAST)"},{"key":"10620_CR32","doi-asserted-by":"publisher","DOI":"10.1002\/hbm.23730","author":"RT Schirrmeister","year":"2017","unstructured":"Schirrmeister RT, Springenberg JT, Fiederer LDJ, Glasstetter M, Eggensperger K, Tangermann M, Hutter F, Burgard W, Ball T (2017) Deep learning with convolutional neural networks for eeg decoding and visualization. Hum Brain Mapp. https:\/\/doi.org\/10.1002\/hbm.23730","journal-title":"Hum Brain Mapp"},{"issue":"4","key":"10620_CR33","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3084225","volume":"50","author":"T Storer","year":"2017","unstructured":"Storer T (2017) Bridging the chasm: A survey of software engineering practice in scientific programming. ACM Computing Surveys (CSUR) 50(4):1\u201332","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"10620_CR34","doi-asserted-by":"publisher","first-page":"134","DOI":"10.1007\/978-3-540-79124-9_10","volume-title":"Tests and proofs","author":"N Tillmann","year":"2008","unstructured":"Tillmann N, de Halleux J (2008) Pex-white box test generation for.net. In: Beckert B, H\u00e4hnle R (eds) Tests and proofs. Springer, Berlin Heidelberg, Berlin, Heidelberg, pp 134\u2013153"},{"key":"10620_CR35","doi-asserted-by":"publisher","unstructured":"Tzoref-Brill R, Sinha S, Nassar AA, Goldin V, Kermany H (2022) Tackletest: a tool for amplifying test generation via type-based combinatorial coverage. In: 2022 IEEE Conference on Software Testing, Verification and Validation (ICST), pp 444\u2013455. https:\/\/doi.org\/10.1109\/ICST53961.2022.00050","DOI":"10.1109\/ICST53961.2022.00050"},{"key":"10620_CR36","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1007\/978-3-540-69389-5_34","volume-title":"Computational science - ICCS 2008","author":"SA Vilkomir","year":"2008","unstructured":"Vilkomir SA, Swain WT, Poore JH, Clarno KT (2008) Modeling input space for testing scientific computational software: a case study. In: Bubak M, van Albada GD, Dongarra J, Sloot PMA (eds) Computational science - ICCS 2008. Springer, Berlin Heidelberg, pp 291\u2013300"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10620-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10620-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10620-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T13:27:33Z","timestamp":1763645253000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10620-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,10]]},"references-count":36,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2025,5]]}},"alternative-id":["10620"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10620-6","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,3,10]]},"assertion":[{"value":"28 January 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 March 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"None","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}},{"value":"This study received IRB approval from the University of North Florida (IRB#1660344-1).","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Participants had to complete an informed consent form before beginning the survey.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed consent"}}],"article-number":"81"}}