{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T23:14:02Z","timestamp":1771024442359,"version":"3.50.1"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"S16","content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["BMC Bioinformatics"],"published-print":{"date-parts":[[2014,12]]},"DOI":"10.1186\/1471-2105-15-s16-s15","type":"journal-article","created":{"date-parts":[[2014,12,8]],"date-time":"2014-12-08T12:03:33Z","timestamp":1418040213000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":71,"title":["Verification and validation of bioinformatics software without a gold standard: a case study of BWA and Bowtie"],"prefix":"10.1186","volume":"15","author":[{"given":"Eleni","family":"Giannoulatou","sequence":"first","affiliation":[]},{"given":"Shin-Ho","family":"Park","sequence":"additional","affiliation":[]},{"given":"David T","family":"Humphreys","sequence":"additional","affiliation":[]},{"given":"Joshua WK","family":"Ho","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2014,12,8]]},"reference":[{"issue":"3","key":"6748_CR1","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1016\/j.tig.2007.12.007","volume":"24","author":"ER Mardis","year":"2008","unstructured":"Mardis ER: The impact of next-generation sequencing technology on genetics. Trends in Genetics. 2008, 24 (3): 133-141. 10.1016\/j.tig.2007.12.007.","journal-title":"Trends in Genetics"},{"issue":"11","key":"6748_CR2","doi-asserted-by":"publisher","first-page":"745","DOI":"10.1038\/nrg3031","volume":"12","author":"MJ Bamshad","year":"2011","unstructured":"Bamshad MJ, Ng SB, Bigham AW, Tabor HK, Emond MJ, Nickerson DA, Shendure J: Exome sequencing as a tool for mendelian disease gene discovery. Nature Reviews Genetics. 2011, 12 (11): 745-755. 10.1038\/nrg3031.","journal-title":"Nature Reviews Genetics"},{"issue":"6","key":"6748_CR3","doi-asserted-by":"publisher","first-page":"623","DOI":"10.1038\/ng.2303","volume":"44","author":"A Kiezun","year":"2012","unstructured":"Kiezun A, Garimella K, Do R, Stitziel NO, Neale BM, McLaren PJ, Gupta N, Sklar P, Sullivan PF, Moran JL: Exome sequencing and the genetic basis of complex traits. Nature Genetics. 2012, 44 (6): 623-630. 10.1038\/ng.2303.","journal-title":"Nature Genetics"},{"issue":"10","key":"6748_CR4","doi-asserted-by":"publisher","first-page":"681","DOI":"10.1038\/nrg3555","volume":"14","author":"KM Boycott","year":"2013","unstructured":"Boycott KM, Vanstone MR, Bulman DE, MacKenzie AE: Rare-disease genetics in the era of next-generation sequencing: discovery to translation. Nature Reviews Genetics. 2013, 14 (10): 681-691. 10.1038\/nrg3555.","journal-title":"Nature Reviews Genetics"},{"issue":"4","key":"6748_CR5","doi-asserted-by":"publisher","first-page":"340","DOI":"10.1056\/NEJMra0907178","volume":"364","author":"WG Feero","year":"2011","unstructured":"Feero WG, Guttmacher AE, McDermott U, Downing JR, Stratton MR: Genomics and the continuum of cancer care. New England Journal of Medicine. 2011, 364 (4): 340-350. 10.1056\/NEJMra0907178.","journal-title":"New England Journal of Medicine"},{"issue":"5","key":"6748_CR6","doi-asserted-by":"publisher","first-page":"333","DOI":"10.1038\/nrg3433","volume":"14","author":"B Berger","year":"2013","unstructured":"Berger B, Peng J, Singh M: Computational solutions for omics data. Nature Reviews Genetics. 2013, 14 (5): 333-346. 10.1038\/nrg3433.","journal-title":"Nature Reviews Genetics"},{"issue":"3","key":"6748_CR7","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1186\/gm432","volume":"5","author":"J O'Rawe","year":"2013","unstructured":"O'Rawe J, Jiang T, Sun G, Wu Y, Wang W, Hu J, Bodily P, Tian L, Hakonarson H, Johnson WE: Low concordance of multiple variant-calling pipelines: practical implications for exome and genome sequencing. Genome Medicine. 2013, 5 (3): 28-10.1186\/gm432.","journal-title":"Genome Medicine"},{"key":"6748_CR8","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1186\/gm543","volume":"6","author":"DJ McCarthy","year":"2014","unstructured":"McCarthy DJ, Humburg P, Kanapin A, Rivas MA, Gaulton K, Consortium TW, Cazier JB, Donnelly P: Choice of transcripts and software has a large effect on variant annotation. Genome Medicine. 2014, 6: 26-10.1186\/gm543.","journal-title":"Genome Medicine"},{"issue":"10","key":"6748_CR9","doi-asserted-by":"publisher","first-page":"785","DOI":"10.1109\/32.328993","volume":"20","author":"L Hatton","year":"1994","unstructured":"Hatton L, Roberts A: How accurate is scientific software?. IEEE Transactions on Software Engineering. 1994, 20 (10): 785-797. 10.1109\/32.328993.","journal-title":"IEEE Transactions on Software Engineering"},{"issue":"6134","key":"6748_CR10","doi-asserted-by":"publisher","first-page":"814","DOI":"10.1126\/science.1231535","volume":"340","author":"LN Joppa","year":"2013","unstructured":"Joppa LN, McInerny G, Harper R, Salido L, Takeda K, O'Hara K, Gavaghan D, Emmott S: Troubling trends in scientific software use. Science. 2013, 340 (6134): 814-815. 10.1126\/science.1231535.","journal-title":"Science"},{"issue":"6060","key":"6748_CR11","doi-asserted-by":"publisher","first-page":"1226","DOI":"10.1126\/science.1213847","volume":"334","author":"RD Peng","year":"2011","unstructured":"Peng RD: Reproducible research in computational science. Science. 2011, 334 (6060): 1226-10.1126\/science.1213847.","journal-title":"Science"},{"issue":"7468","key":"6748_CR12","doi-asserted-by":"publisher","first-page":"472","DOI":"10.1038\/501472a","volume":"501","author":"EC Hayden","year":"2013","unstructured":"Hayden EC: Mozilla plan seeks to debug scientific code. Nature. 2013, 501 (7468): 472-472. 10.1038\/501472a.","journal-title":"Nature"},{"issue":"9","key":"6748_CR13","doi-asserted-by":"publisher","first-page":"87","DOI":"10.1371\/journal.pcbi.0020087","volume":"2","author":"SM Baxter","year":"2006","unstructured":"Baxter SM, Day SW, Fetrow JS, Reisinger SJ: Scientific software development is not an oxymoron. PLoS Computational Biology. 2006, 2 (9): 87-10.1371\/journal.pcbi.0020087.","journal-title":"PLoS Computational Biology"},{"issue":"10","key":"6748_CR14","doi-asserted-by":"publisher","first-page":"1003285","DOI":"10.1371\/journal.pcbi.1003285","volume":"9","author":"GK Sandve","year":"2013","unstructured":"Sandve GK, Nekrutenko A, Taylor J, Hovig E: Ten simple rules for reproducible computational research. PLoS Computational Biology. 2013, 9 (10): 1003285-10.1371\/journal.pcbi.1003285.","journal-title":"PLoS Computational Biology"},{"issue":"1","key":"6748_CR15","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1186\/2047-217X-2-15","volume":"2","author":"T Seemann","year":"2013","unstructured":"Seemann T: Ten recommendations for creating usable bioinformatics command line software. GigaScience. 2013, 2 (1): 15-10.1186\/2047-217X-2-15.","journal-title":"GigaScience"},{"issue":"10","key":"6748_CR16","doi-asserted-by":"publisher","first-page":"894","DOI":"10.1038\/nbt.2721","volume":"31","author":"S Altschul","year":"2013","unstructured":"Altschul S, Demchak B, Durbin R, Gentleman R, Krzywinski M, Li H, Nekrutenko A, Robinson J, Rasband W, Taylor J: The anatomy of successful computational biology software. Nature Biotechnology. 2013, 31 (10): 894-897. 10.1038\/nbt.2721.","journal-title":"Nature Biotechnology"},{"issue":"1","key":"6748_CR17","doi-asserted-by":"publisher","first-page":"1001745","DOI":"10.1371\/journal.pbio.1001745","volume":"12","author":"G Wilson","year":"2014","unstructured":"Wilson G, Aruliah D, Brown CT, Hong NPC, Davis M, Guy RT, Haddock SH, Huff KD, Mitchell IM, Plumbley MD: Best practices for scientific computing. PLoS Biology. 2014, 12 (1): 1001745-10.1371\/journal.pbio.1001745.","journal-title":"PLoS Biology"},{"key":"6748_CR18","volume-title":"Software Testing Techniques","author":"B Beizer","year":"1990","unstructured":"Beizer B: Software Testing Techniques. 1990, Van Nostrand Reinhold"},{"key":"6748_CR19","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809163","volume-title":"Introduction to Software Testing","author":"P Ammann","year":"2008","unstructured":"Ammann P, Offutt J: Introduction to Software Testing. 2008, Cambridge University Press"},{"key":"6748_CR20","first-page":"85","volume-title":"2007 Future of Software Engineering","author":"A Bertolino","year":"2007","unstructured":"Bertolino A: Software testing research: Achievements, challenges, dreams. 2007 Future of Software Engineering. 2007, IEEE Computer Society, 85-103."},{"key":"6748_CR21","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1093\/bioinformatics\/btm566","volume":"24","author":"TW Evans","year":"2008","unstructured":"Evans TW, Gillespie CS, Wilkinson DJ: The SBML discrete stochastic models test suite. Bioinformatics. 2008, 24: 285-286. 10.1093\/bioinformatics\/btm566.","journal-title":"Bioinformatics"},{"key":"6748_CR22","doi-asserted-by":"publisher","first-page":"1963","DOI":"10.1093\/bioinformatics\/btn319","volume":"24","author":"FT Bergmann","year":"2008","unstructured":"Bergmann FT, Sauro HM: Comparing simulation results of SBML capable simulators. Bioinformatics. 2008, 24: 1963-1965. 10.1093\/bioinformatics\/btn319.","journal-title":"Bioinformatics"},{"key":"6748_CR23","volume-title":"Technical Report HKUST-CS98-01","author":"TY Chen","year":"1998","unstructured":"Chen TY, Cheung SC, Yiu SM: Metamorphic testing: a new approach for generating next test cases. Technical Report HKUST-CS98-01. 1998, Department of Computer Science, Hong Kong University of Science and Technology, Hong Kong"},{"key":"6748_CR24","first-page":"327","volume-title":"Proceedings of the 26th Annual International Computer Software and Applications Conference (COMPSAC 2002)","author":"TY Chen","year":"2002","unstructured":"Chen TY, Feng J, Tse TH: Metamorphic testing of programs on partial differential equations: a case study. Proceedings of the 26th Annual International Computer Software and Applications Conference (COMPSAC 2002). 2002, IEEE Computer Society Press, 327-333."},{"key":"6748_CR25","first-page":"569","volume-title":"Proceedings of the 4th Ibero-American Symposium on Software Engineering and Knowledge Engineering (JIISIC 2004)","author":"TY Chen","year":"2004","unstructured":"Chen TY, Huang DH, Tse TH, Zhou ZQ: Case studies on the selection of useful relations in metamorphic testing. Proceedings of the 4th Ibero-American Symposium on Software Engineering and Knowledge Engineering (JIISIC 2004). 2004, Polytechnic University of Madrid, Madrid, Spain, 569-583."},{"key":"6748_CR26","volume-title":"Proceedings of the First Workshop on End-User Software Engineering","author":"TY Chen","year":"2005","unstructured":"Chen TY, Kuo FC, Zhou ZQ: An effective testing method for end-user programmer. Proceedings of the First Workshop on End-User Software Engineering. 2005"},{"issue":"1","key":"6748_CR27","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1186\/1471-2105-10-24","volume":"10","author":"TY Chen","year":"2009","unstructured":"Chen TY, Ho JW, Liu H, Xie X: An innovative approach for testing bioinformatics programs using metamorphic testing. BMC Bioinformatics. 2009, 10 (1): 24-10.1186\/1471-2105-10-24.","journal-title":"BMC Bioinformatics"},{"issue":"06","key":"6748_CR28","doi-asserted-by":"publisher","first-page":"729","DOI":"10.1142\/S021972001100563X","volume":"09","author":"MS Sadi","year":"2011","unstructured":"Sadi MS, Kuo F, Ho JWK, Charleston MA, Chen TY: Verification of phylogenetic inference programs using metamorphic testing. Journal of Bioinformatics and Computational Biology. 2011, 09 (06): 729-747. 10.1142\/S021972001100563X.","journal-title":"Journal of Bioinformatics and Computational Biology"},{"issue":"4","key":"6748_CR29","doi-asserted-by":"publisher","first-page":"544","DOI":"10.1016\/j.jss.2010.11.920","volume":"84","author":"X Xie","year":"2011","unstructured":"Xie X, Ho JWK, Murphy C, Kaiser G, Xu B, Chen TY: Testing and validating machine learning classifiers by metamorphic testing. Journal of Systems and Software. 2011, 84 (4): 544-558. 10.1016\/j.jss.2010.11.920.","journal-title":"Journal of Systems and Software"},{"issue":"14","key":"6748_CR30","doi-asserted-by":"publisher","first-page":"1754","DOI":"10.1093\/bioinformatics\/btp324","volume":"25","author":"H Li","year":"2009","unstructured":"Li H, Durbin R: Fast and accurate short read alignment with burrows-wheeler transform. Bioinformatics. 2009, 25 (14): 1754-1760. 10.1093\/bioinformatics\/btp324.","journal-title":"Bioinformatics"},{"issue":"3","key":"6748_CR31","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1186\/gb-2009-10-3-r25","volume":"10","author":"B Langmead","year":"2009","unstructured":"Langmead B, Trapnell C, Pop M, Salzberg SL: Ultrafast and memory-efficient alignment of short dna sequences to the human genome. Genome Biology. 2009, 10 (3): 25-10.1186\/gb-2009-10-3-r25.","journal-title":"Genome Biology"},{"issue":"4","key":"6748_CR32","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1038\/nmeth.1923","volume":"9","author":"B Langmead","year":"2012","unstructured":"Langmead B, Salzberg SL: Fast gapped-read alignment with bowtie 2. Nature Methods. 2012, 9 (4): 357-359. 10.1038\/nmeth.1923.","journal-title":"Nature Methods"},{"issue":"20","key":"6748_CR33","doi-asserted-by":"publisher","first-page":"2790","DOI":"10.1093\/bioinformatics\/btr477","volume":"27","author":"M Ruffalo","year":"2011","unstructured":"Ruffalo M, LaFramboise T, Koyut\u00fcrk M: Comparative analysis of algorithms for next-generation sequencing read alignment. Bioinformatics. 2011, 27 (20): 2790-2796. 10.1093\/bioinformatics\/btr477.","journal-title":"Bioinformatics"},{"issue":"5","key":"6748_CR34","doi-asserted-by":"publisher","first-page":"473","DOI":"10.1093\/bib\/bbq015","volume":"11","author":"H Li","year":"2010","unstructured":"Li H, Homer N: A survey of sequence alignment algorithms for next-generation sequencing. Briefings in Bioinformatics. 2010, 11 (5): 473-483. 10.1093\/bib\/bbq015.","journal-title":"Briefings in Bioinformatics"},{"issue":"16","key":"6748_CR35","doi-asserted-by":"publisher","first-page":"2078","DOI":"10.1093\/bioinformatics\/btp352","volume":"25","author":"H Li","year":"2009","unstructured":"Li H, Handsaker B, Wysoker A, Fennell T, Ruan J, Homer N, Marth G, Abecasis G, Durbin R: The sequence alignment\/map format and samtools. Bioinformatics. 2009, 25 (16): 2078-2079. 10.1093\/bioinformatics\/btp352.","journal-title":"Bioinformatics"},{"issue":"7319","key":"6748_CR36","doi-asserted-by":"publisher","first-page":"1061","DOI":"10.1038\/nature09534","volume":"467","author":"1000 Genomes Project Consortium","year":"2010","unstructured":"1000 Genomes Project Consortium: A map of human genome variation from population-scale sequencing. Nature. 2010, 467 (7319): 1061-1073. 10.1038\/nature09534.","journal-title":"Nature"},{"issue":"7422","key":"6748_CR37","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1038\/nature11632","volume":"491","author":"1000 Genomes Project Consortium","year":"2012","unstructured":"1000 Genomes Project Consortium: An integrated map of genetic variation from 1,092 human genomes. Nature. 2012, 491 (7422): 56-65. 10.1038\/nature11632.","journal-title":"Nature"},{"issue":"9","key":"6748_CR38","doi-asserted-by":"publisher","first-page":"1297","DOI":"10.1101\/gr.107524.110","volume":"20","author":"A McKenna","year":"2010","unstructured":"McKenna A, Hanna M, Banks E, Sivachenko A, Cibulskis K, Kernytsky A, Garimella K, Altshuler D, Gabriel S, Daly M: The genome analysis toolkit: a mapreduce framework for analyzing next-generation dna sequencing data. Genome Research. 2010, 20 (9): 1297-1303. 10.1101\/gr.107524.110.","journal-title":"Genome Research"},{"issue":"5","key":"6748_CR39","doi-asserted-by":"publisher","first-page":"473","DOI":"10.1093\/bib\/bbq015","volume":"11","author":"H Li","year":"2010","unstructured":"Li H, Homer N: A survey of sequence alignment algorithms for next-generation sequencing. Briefings in Bioinformatics. 2010, 11 (5): 473-483. 10.1093\/bib\/bbq015.","journal-title":"Briefings in Bioinformatics"},{"issue":"6","key":"6748_CR40","doi-asserted-by":"publisher","first-page":"406","DOI":"10.1038\/jhg.2011.43","volume":"56","author":"S Bao","year":"2011","unstructured":"Bao S, Jiang R, Kwan W, Wang B, Ma X, Song YQ: Evaluation of next-generation sequencing software in mapping and assembly. Journal of Human Genetics. 2011, 56 (6): 406-414. 10.1038\/jhg.2011.43.","journal-title":"Journal of Human Genetics"},{"issue":"24","key":"6748_CR41","doi-asserted-by":"publisher","first-page":"3169","DOI":"10.1093\/bioinformatics\/bts605","volume":"28","author":"NA Fonseca","year":"2012","unstructured":"Fonseca NA, Rung J, Brazma A, Marioni JC: Tools for mapping high-throughput sequencing data. Bioinformatics. 2012, 28 (24): 3169-3177. 10.1093\/bioinformatics\/bts605.","journal-title":"Bioinformatics"},{"issue":"1","key":"6748_CR42","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1186\/1756-0381-5-6","volume":"5","author":"X Yu","year":"2012","unstructured":"Yu X, Guda K, Willis J, Veigl M, Wang Z, Markowitz S, Adams MD, Sun S: How do alignment programs perform on sequencing data with varying qualities and from repetitive regions?. BioData Mining. 2012, 5 (1): 6-10.1186\/1756-0381-5-6.","journal-title":"BioData Mining"}],"container-title":["BMC Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/1471-2105-15-S16-S15.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,9,2]],"date-time":"2021-09-02T18:55:28Z","timestamp":1630608928000},"score":1,"resource":{"primary":{"URL":"https:\/\/bmcbioinformatics.biomedcentral.com\/articles\/10.1186\/1471-2105-15-S16-S15"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,12]]},"references-count":42,"journal-issue":{"issue":"S16","published-print":{"date-parts":[[2014,12]]}},"alternative-id":["6748"],"URL":"https:\/\/doi.org\/10.1186\/1471-2105-15-s16-s15","relation":{},"ISSN":["1471-2105"],"issn-type":[{"value":"1471-2105","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,12]]},"assertion":[{"value":"8 December 2014","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"S15"}}