{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,22]],"date-time":"2026-07-22T22:29:58Z","timestamp":1784759398139,"version":"3.55.0"},"reference-count":30,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2022,7,26]],"date-time":"2022-07-26T00:00:00Z","timestamp":1658793600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,7,26]],"date-time":"2022-07-26T00:00:00Z","timestamp":1658793600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000083","name":"Directorate for Computer and Information Science and Engineering","doi-asserted-by":"publisher","award":["1445344"],"award-info":[{"award-number":["1445344"]}],"id":[{"id":"10.13039\/100000083","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2022,11]]},"DOI":"10.1007\/s10664-022-10184-9","type":"journal-article","created":{"date-parts":[[2022,7,26]],"date-time":"2022-07-26T09:05:01Z","timestamp":1658826301000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Testing research software: a survey"],"prefix":"10.1007","volume":"27","author":[{"given":"Nasir U.","family":"Eisty","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7824-9151","authenticated-orcid":false,"given":"Jeffrey C.","family":"Carver","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,7,26]]},"reference":[{"issue":"4","key":"10184_CR1","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1109\/MS.2008.93","volume":"25","author":"KS Ackroyd","year":"2008","unstructured":"Ackroyd K S, Kinder S H, Mant G R, Miller M C, Ramsdale C A, Stephenson P C (2008) Scientific software development at a research facility. IEEE Softw 25(4):44\u201351. https:\/\/doi.org\/10.1109\/MS.2008.93","journal-title":"IEEE Softw"},{"key":"10184_CR2","doi-asserted-by":"publisher","DOI":"10.1017\/9781316771273","volume-title":"Introduction to software testing, 2nd edn","author":"P Ammann","year":"2016","unstructured":"Ammann P, Offutt J (2016) Introduction to software testing, 2nd edn. Cambridge University Press, Cambridge"},{"key":"10184_CR3","unstructured":"Bourque P, Fairley RE (eds) (2014) SWEBOK: guide to the software engineering body of knowledge, version 3.0 edn. IEEE Computer Society, Los Alamitos. http:\/\/www.swebok.org\/"},{"key":"10184_CR4","doi-asserted-by":"publisher","unstructured":"Carver JC, Eisty N (2021) Testing research software: survey data. https:\/\/doi.org\/10.6084\/m9.figshare.16663561. https:\/\/figshare.com\/articles\/dataset\/_\/16663561\/0","DOI":"10.6084\/m9.figshare.16663561"},{"issue":"6","key":"10184_CR5","doi-asserted-by":"publisher","first-page":"49","DOI":"10.1109\/MS.2011.117","volume":"28","author":"T Clune","year":"2011","unstructured":"Clune T, Rood R (2011) Software testing and verification in climate model development. IEEE Softw 28(6):49\u201355. https:\/\/doi.org\/10.1109\/MS.2011.117","journal-title":"IEEE Softw"},{"issue":"3","key":"10184_CR6","doi-asserted-by":"publisher","first-page":"177","DOI":"10.1177\/1094342005056094","volume":"19","author":"JB Drake","year":"2005","unstructured":"Drake J B, Jones P W, George R Carr J (2005) Overview of the software design of the community climate system model. Int J High Perform Comput Appl 19 (3):177\u2013186. https:\/\/doi.org\/10.1177\/1094342005056094https:\/\/doi.org\/10.1177\/1094342005056094","journal-title":"Int J High Perform Comput Appl"},{"key":"10184_CR7","doi-asserted-by":"publisher","unstructured":"Easterbrook S M (2010) Climate change: a grand software challenge. In: Proceedings of the FSE\/SDP workshop on future of software engineering research, FoSER \u201910. https:\/\/doi.org\/10.1145\/1882362.1882383. http:\/\/doi.acm.org\/10.1145\/1882362.1882383. ACM, pp 99\u2013104","DOI":"10.1145\/1882362.1882383"},{"issue":"6","key":"10184_CR8","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1109\/MCSE.2009.193","volume":"11","author":"SM Easterbrook","year":"2009","unstructured":"Easterbrook S M, Johns T C (2009) Engineering the software for understanding climate change. Comput Sci Eng 11(6):65\u201374. https:\/\/doi.org\/10.1109\/MCSE.2009.193","journal-title":"Comput Sci Eng"},{"issue":"6","key":"10184_CR9","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1109\/MCSE.2009.186","volume":"11","author":"SL Eddins","year":"2009","unstructured":"Eddins S L (2009) Automated software testing for matlab. Comput Sci Eng 11(6):48\u201355. https:\/\/doi.org\/10.1109\/MCSE.2009.186https:\/\/doi.org\/10.1109\/MCSE.2009.186","journal-title":"Comput Sci Eng"},{"key":"10184_CR10","doi-asserted-by":"publisher","unstructured":"Eisty N U, Thiruvathukal G K, Carver J C (2018) A survey of software metric use in research software development. In: 2018 IEEE 14th international conference on e-science (e-science). https:\/\/doi.org\/10.1109\/eScience.2018.00036, pp 212\u2013222","DOI":"10.1109\/eScience.2018.00036"},{"issue":"2","key":"10184_CR11","doi-asserted-by":"publisher","first-page":"435","DOI":"10.5194\/gmd-4-435-2011","volume":"4","author":"PE Farrell","year":"2011","unstructured":"Farrell P E, Piggott M D, Gorman G J, Ham D A, Wilson C R, Bond T M (2011) Automated continuous verification for numerical simulation. Geosci Model Dev 4(2):435\u2013449. https:\/\/doi.org\/10.5194\/gmd-4-435-2011https:\/\/doi.org\/10.5194\/gmd-4-435-2011. https:\/\/www.geosci-model-dev.net\/4\/435\/2011\/","journal-title":"Geosci Model Dev"},{"key":"10184_CR12","doi-asserted-by":"publisher","unstructured":"Hannay J E, MacLeod C, Singer J, Langtangen H P, Pfahl D, Wilson G (2009) How do scientists develop and use scientific software?. In: 2009 ICSE workshop on software engineering for computational science and engineering. https:\/\/doi.org\/10.1109\/SECSE.2009.5069155, pp 1\u20138","DOI":"10.1109\/SECSE.2009.5069155"},{"key":"10184_CR13","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1016\/j.infsof.2015.07.011","volume":"67","author":"D Heaton","year":"2015","unstructured":"Heaton D, Carver J C (2015) Claims about the use of software engineering practices in science: a systematic literature review. Inf Softw Tech 67:207\u2013219. https:\/\/doi.org\/10.1016\/j.infsof.2015.07.011. http:\/\/www.sciencedirect.com\/science\/article\/pii\/S0950584915001342","journal-title":"Inf Softw Tech"},{"key":"10184_CR14","doi-asserted-by":"publisher","unstructured":"Hill C (2016) Socio-economic status and computer use: designing software that supports low-income users. In: 2016 IEEE Symposium on visual languages and human-centric computing. https:\/\/doi.org\/10.1109\/VLHCC.2016.7739651, pp 1\u20131","DOI":"10.1109\/VLHCC.2016.7739651"},{"issue":"3","key":"10184_CR15","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1109\/MC.2008.101","volume":"41","author":"L Hochstein","year":"2008","unstructured":"Hochstein L, Basili V R (2008) The asc-alliance projects: a case study of large-scale parallel scientific code development. Computer 41(3):50\u201358. https:\/\/doi.org\/10.1109\/MC.2008.101","journal-title":"Computer"},{"key":"10184_CR16","doi-asserted-by":"publisher","unstructured":"Hook D, Kelly D (2009) Testing for trustworthiness in scientific software. In: 2009 ICSE Workshop on software engineering for computational science and engineering. https:\/\/doi.org\/10.1109\/SECSE.2009.5069163https:\/\/doi.org\/10.1109\/SECSE.2009.5069163, pp 59\u201364","DOI":"10.1109\/SECSE.2009.5069163 10.1109\/SECSE.2009.5069163"},{"issue":"10","key":"10184_CR17","doi-asserted-by":"publisher","first-page":"1219","DOI":"10.1016\/j.infsof.2014.05.006","volume":"56","author":"U Kanewala","year":"2014","unstructured":"Kanewala U, Bieman J M (2014) Testing scientific software: a systematic literature review. Inf Softw Technol 56(10):1219\u20131232. https:\/\/doi.org\/10.1016\/j.infsof.2014.05.006","journal-title":"Inf Softw Technol"},{"issue":"2","key":"10184_CR18","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1109\/MCSE.2018.2883051","volume":"21","author":"DS Katz","year":"2019","unstructured":"Katz D S, McInnes L C, Bernholdt D E, Mayes A C, Hong N P C, Duckles J, Gesing S, Heroux M A, Hettrick S, Jimenez R C, Pierce M, Weaver B, Wilkins-Diehr N (2019) Community organizations: changing the culture in which research software is developed and sustained. Comput Sci Eng 21(2):8\u201324. https:\/\/doi.org\/10.1109\/MCSE.2018.2883051","journal-title":"Comput Sci Eng"},{"key":"10184_CR19","unstructured":"Kelly D, Sanders R, Saint R, Floor P, Sanders R, Kelly D (2008) The challenge of testing scientific software. In: Proceedings of the conference for the association for software testing, pp 30\u201336"},{"issue":"5","key":"10184_CR20","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1109\/MCSE.2009.139","volume":"11","author":"D Kelly","year":"2009","unstructured":"Kelly D, Hook D, Sanders R (2009) Five recommended practices for computational scientists who write software. Comput Sci Eng 11(5):48\u201353. https:\/\/doi.org\/10.1109\/MCSE.2009.139","journal-title":"Comput Sci Eng"},{"issue":"3","key":"10184_CR21","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1109\/MS.2010.88","volume":"28","author":"D Kelly","year":"2011","unstructured":"Kelly D, Thorsteinson S, Hook D (2011) Scientific software testing: analysis with four dimensions. IEEE Softw 28(3):84\u201390. https:\/\/doi.org\/10.1109\/MS.2010.88","journal-title":"IEEE Softw"},{"key":"10184_CR22","doi-asserted-by":"publisher","unstructured":"Lemos G S, Martins E (2012) Specification-guided golden run for analysis of robustness testing results. In: 2012 IEEE Sixth international conference on software security and reliability. https:\/\/doi.org\/10.1109\/SERE.2012.28, pp 157\u2013166","DOI":"10.1109\/SERE.2012.28"},{"issue":"5807","key":"10184_CR23","doi-asserted-by":"publisher","first-page":"1856","DOI":"10.1126\/science.314.5807.1856","volume":"314","author":"G Miller","year":"2006","unstructured":"Miller G (2006) A scientist\u2019s nightmare: software problem leads to five retractions. Science 314(5807):1856\u20131857. https:\/\/doi.org\/10.1126\/science.314.5807.1856. https:\/\/science.sciencemag.org\/content\/314\/5807\/1856, https:\/\/science.sciencemag.org\/content\/314\/5807\/1856.full.pdf","journal-title":"Science"},{"key":"10184_CR24","doi-asserted-by":"publisher","unstructured":"Murphy C, Raunak M, King A, Chen S, Imbraino C, Kaiser G, Lee I, Sokolsky O, Clarke L, Osterweil L (2011) On effective testing of health care simulation software. Technical Reports (CIS). https:\/\/doi.org\/10.1145\/1987993.1988003","DOI":"10.1145\/1987993.1988003"},{"key":"10184_CR25","doi-asserted-by":"publisher","unstructured":"Nguyen-Hoan L, Flint S, Sankaranarayana R (2010) A survey of scientific software development. In: Proceedings of the 2010 ACM-IEEE international symposium on empirical software engineering and measurement, ESEM \u201910. https:\/\/doi.org\/10.1145\/1852786.1852802. http:\/\/doi.acm.org\/10.1145\/1852786.1852802. ACM, pp 12:1\u201312:10","DOI":"10.1145\/1852786.1852802"},{"issue":"4","key":"10184_CR26","doi-asserted-by":"publisher","first-page":"399","DOI":"10.1177\/1094342004048534","volume":"18","author":"DE Post","year":"2004","unstructured":"Post D E, Kendall R P (2004) Software project management and quality engineering practices for complex, coupled multiphysics, massively parallel computational simulations: Lessons learned from asci. Int J High Perform Comput Appl 18(4):399\u2013416. https:\/\/doi.org\/10.1177\/1094342004048534","journal-title":"Int J High Perform Comput Appl"},{"issue":"4","key":"10184_CR27","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1109\/MS.2008.84","volume":"25","author":"R Sanders","year":"2008","unstructured":"Sanders R, Kelly D (2008) Dealing with risk in scientific software development. IEEE Softw 25(4):21\u201328. https:\/\/doi.org\/10.1109\/MS.2008.84","journal-title":"IEEE Softw"},{"key":"10184_CR28","doi-asserted-by":"publisher","unstructured":"Segal J (2005) When software engineers met research scientists: a case study. Empir Softw Eng 10. https:\/\/doi.org\/10.1007\/s10664-005-3865-y","DOI":"10.1007\/s10664-005-3865-y"},{"issue":"5","key":"10184_CR29","doi-asserted-by":"publisher","first-page":"581","DOI":"10.1007\/s10606-009-9096-9","volume":"18","author":"J Segal","year":"2009","unstructured":"Segal J (2009) Software development cultures and cooperation problems: a field study of the early stages of development of software for a scientific community. Comput Supported Coop Work 18 (5):581. https:\/\/doi.org\/10.1007\/s10606-009-9096-9","journal-title":"Comput Supported Coop Work"},{"key":"10184_CR30","doi-asserted-by":"crossref","unstructured":"Vilkomir S A, Swain W T, Poore J H, Clarno K T (2008) Modeling input space for testing scientific computational software: a case study. In: Bubak M, van Albada GD, Dongarra J, Sloot PMA (eds) Computational science\u2014ICCS 2008. Springer, Berlin, pp 291\u2013300","DOI":"10.1007\/978-3-540-69389-5_34"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-022-10184-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-022-10184-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-022-10184-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,26]],"date-time":"2022-09-26T08:24:21Z","timestamp":1664180661000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-022-10184-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,26]]},"references-count":30,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2022,11]]}},"alternative-id":["10184"],"URL":"https:\/\/doi.org\/10.1007\/s10664-022-10184-9","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,7,26]]},"assertion":[{"value":"26 May 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 July 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"None","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Conflict of Interest"}}],"article-number":"138"}}