{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T16:57:28Z","timestamp":1775581048373,"version":"3.50.1"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1007349","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,9,13]],"date-time":"2019-09-13T00:00:00Z","timestamp":1568332800000}}],"reference-count":60,"publisher":"Public Library of Science (PLoS)","issue":"9","license":[{"start":{"date-parts":[[2019,9,3]],"date-time":"2019-09-03T00:00:00Z","timestamp":1567468800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1007349","type":"journal-article","created":{"date-parts":[[2019,9,3]],"date-time":"2019-09-03T15:20:53Z","timestamp":1567524053000},"page":"e1007349","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":76,"title":["Evaluation of parameters affecting performance and reliability of machine learning-based antibiotic susceptibility testing from whole genome sequencing data"],"prefix":"10.1371","volume":"15","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1372-1301","authenticated-orcid":true,"given":"Allison L.","family":"Hicks","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4599-9164","authenticated-orcid":true,"given":"Nicole","family":"Wheeler","sequence":"additional","affiliation":[]},{"given":"Leonor","family":"S\u00e1nchez-Bus\u00f3","sequence":"additional","affiliation":[]},{"given":"Jennifer L.","family":"Rakeman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1512-6194","authenticated-orcid":true,"given":"Simon R.","family":"Harris","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5646-1314","authenticated-orcid":true,"given":"Yonatan H.","family":"Grad","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2019,9,3]]},"reference":[{"key":"pcbi.1007349.ref001","article-title":"The Review on Antimicrobial Resistance","year":"2016","journal-title":"Tackling drug-resistant infections globally: final report and recommendations"},{"issue":"11","key":"pcbi.1007349.ref002","doi-asserted-by":"crossref","first-page":"1123","DOI":"10.1016\/S1473-3099(14)70827-8","article-title":"Rapid point of care diagnostic tests for viral and bacterial respiratory tract infections\u2014needs, advances, and future prospects","volume":"14","author":"A Zumla","year":"2014","journal-title":"Lancet Infect Dis"},{"issue":"9","key":"pcbi.1007349.ref003","doi-asserted-by":"crossref","first-page":"601","DOI":"10.1038\/nrg3226","article-title":"Transforming clinical microbiology with bacterial genome sequencing","volume":"13","author":"X Didelot","year":"2012","journal-title":"Nat Rev Genet"},{"issue":"10","key":"pcbi.1007349.ref004","doi-asserted-by":"crossref","first-page":"1193","DOI":"10.1016\/S1473-3099(15)00062-6","article-title":"Whole-genome sequencing for prediction of Mycobacterium tuberculosis drug susceptibility and resistance: a retrospective cohort study","volume":"15","author":"TM Walker","year":"2015","journal-title":"Lancet Infect Dis"},{"issue":"8","key":"pcbi.1007349.ref005","doi-asserted-by":"crossref","first-page":"2641","DOI":"10.1128\/JCM.02741-12","article-title":"Rifampin resistance missed in automated liquid culture system for Mycobacterium tuberculosis isolates with specific rpoB mutations","volume":"51","author":"L Rigouts","year":"2013","journal-title":"J Clin Microbiol"},{"issue":"9","key":"pcbi.1007349.ref006","doi-asserted-by":"crossref","DOI":"10.1128\/JCM.01815-17","article-title":"Accuracy of Different Bioinformatics Methods in Detecting Antibiotic Resistance and Virulence Factors from Staphylococcus aureus Whole-Genome Sequences","volume":"56","author":"A Mason","year":"2018","journal-title":"J Clin Microbiol"},{"issue":"10","key":"pcbi.1007349.ref007","doi-asserted-by":"crossref","first-page":"1666","DOI":"10.1093\/bioinformatics\/btx801","article-title":"Machine learning for classifying tuberculosis drug-resistance from DNA sequencing data","volume":"34","author":"Y Yang","year":"2018","journal-title":"Bioinformatics"},{"key":"pcbi.1007349.ref008","doi-asserted-by":"crossref","first-page":"1887","DOI":"10.3389\/fmicb.2016.01887","article-title":"Evaluation of Machine Learning and Rules-Based Approaches for Predicting Antimicrobial Resistance Profiles in Gram-negative Bacilli from Whole Genome Sequence Data","volume":"7","author":"MW Pesesky","year":"2016","journal-title":"Front Microbiol"},{"issue":"1","key":"pcbi.1007349.ref009","doi-asserted-by":"crossref","first-page":"621","DOI":"10.1186\/s12864-017-4017-7","article-title":"Validation of beta-lactam minimum inhibitory concentration predictions for pneumococcal isolates with newly encountered penicillin binding protein (PBP) sequences","volume":"18","author":"Y Li","year":"2017","journal-title":"BMC Genomics"},{"key":"pcbi.1007349.ref010","doi-asserted-by":"crossref","first-page":"10063","DOI":"10.1038\/ncomms10063","article-title":"Rapid antibiotic-resistance predictions from genome sequence data for Staphylococcus aureus and Mycobacterium tuberculosis","volume":"6","author":"P Bradley","year":"2015","journal-title":"Nat Commun"},{"issue":"1","key":"pcbi.1007349.ref011","doi-asserted-by":"crossref","first-page":"754","DOI":"10.1186\/s12864-016-2889-6","article-title":"Predictive computational phenotyping and biomarker discovery using reference-free genome comparisons","volume":"17","author":"A Drouin","year":"2016","journal-title":"BMC Genomics"},{"issue":"1","key":"pcbi.1007349.ref012","doi-asserted-by":"crossref","first-page":"4071","DOI":"10.1038\/s41598-019-40561-2","article-title":"Interpretable genotype-to-phenotype classifiers with performance guarantees","volume":"9","author":"A Drouin","year":"2019","journal-title":"Sci Rep"},{"key":"pcbi.1007349.ref013","doi-asserted-by":"crossref","first-page":"27930","DOI":"10.1038\/srep27930","article-title":"Antimicrobial Resistance Prediction in PATRIC and RAST","volume":"6","author":"JJ Davis","year":"2016","journal-title":"Sci Rep"},{"issue":"1","key":"pcbi.1007349.ref014","doi-asserted-by":"crossref","first-page":"421","DOI":"10.1038\/s41598-017-18972-w","article-title":"Developing an in silico minimum inhibitory concentration panel test for Klebsiella pneumoniae","volume":"8","author":"M Nguyen","year":"2018","journal-title":"Sci Rep"},{"key":"pcbi.1007349.ref015","article-title":"Using machine learning to predict antimicrobial minimum inhibitory concentrations and associated genomic features for nontyphoidal Salmonella","author":"M Nguyen","year":"2018","journal-title":"J Clin Microbiol"},{"key":"pcbi.1007349.ref016","article-title":"Machine Learning for Antimicrobial Resistance","author":"JW Santerre","year":"2016","journal-title":"arXiv e-prints"},{"issue":"12","key":"pcbi.1007349.ref017","doi-asserted-by":"crossref","first-page":"e1006258","DOI":"10.1371\/journal.pcbi.1006258","article-title":"Prediction of antibiotic resistance in Escherichia coli from large-scale pan-genome data","volume":"14","author":"D Moradigaravand","year":"2018","journal-title":"PLoS Comput Biol"},{"issue":"4","key":"pcbi.1007349.ref018","doi-asserted-by":"crossref","first-page":"1182","DOI":"10.1128\/JCM.03117-13","article-title":"Prediction of Staphylococcus aureus antimicrobial resistance by whole-genome sequencing","volume":"52","author":"NC Gordon","year":"2014","journal-title":"J Clin Microbiol"},{"key":"pcbi.1007349.ref019","first-page":"723","article-title":"The set covering machine","volume":"3","author":"M Marchland","year":"2002","journal-title":"Journal of Machine Learning Research"},{"key":"pcbi.1007349.ref020","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests","volume":"45","author":"L. Breiman","year":"2001","journal-title":"Machine Learning"},{"issue":"3","key":"pcbi.1007349.ref021","doi-asserted-by":"crossref","first-page":"805","DOI":"10.1128\/JCM.03032-15","article-title":"Performance and Verification of a Real-Time PCR Assay Targeting the gyrA Gene for Prediction of Ciprofloxacin Resistance in Neisseria gonorrhoeae","volume":"54","author":"P Hemarajata","year":"2016","journal-title":"J Clin Microbiol"},{"issue":"4","key":"pcbi.1007349.ref022","doi-asserted-by":"crossref","first-page":"1250","DOI":"10.1128\/JCM.01909-06","article-title":"Real-time PCR assay for detection of quinolone-resistant Neisseria gonorrhoeae in urine samples","volume":"45","author":"MJ Siedner","year":"2007","journal-title":"J Clin Microbiol"},{"issue":"10","key":"pcbi.1007349.ref023","doi-asserted-by":"crossref","first-page":"1579","DOI":"10.1093\/infdis\/jiw420","article-title":"Genomic Epidemiology of Gonococcal Resistance to Extended-Spectrum Cephalosporins, Macrolides, and Fluoroquinolones in the United States, 2000\u20132013","volume":"214","author":"YH Grad","year":"2016","journal-title":"J Infect Dis"},{"issue":"4","key":"pcbi.1007349.ref024","doi-asserted-by":"crossref","DOI":"10.1128\/mBio.01419-18","article-title":"Azithromycin Resistance through Interspecific Acquisition of an Epistasis-Dependent Efflux Pump Component and Transcriptional Regulator in Neisseria gonorrhoeae","volume":"9","author":"CB Wadsworth","year":"2018","journal-title":"MBio"},{"issue":"6","key":"pcbi.1007349.ref025","doi-asserted-by":"crossref","first-page":"e0218204","DOI":"10.1371\/journal.pone.0218204","article-title":"Comparative genome analysis reveals niche-specific genome expansion in Acinetobacter baumannii strains","volume":"14","author":"H Yakkala","year":"2019","journal-title":"PLoS One"},{"issue":"27","key":"pcbi.1007349.ref026","doi-asserted-by":"crossref","first-page":"E3574","DOI":"10.1073\/pnas.1501049112","article-title":"Genomic analysis of diversity, population structure, virulence, and antimicrobial resistance in Klebsiella pneumoniae, an urgent threat to public health","volume":"112","author":"KE Holt","year":"2015","journal-title":"Proc Natl Acad Sci U S A"},{"key":"pcbi.1007349.ref027","article-title":"Genomics of antibiotic-resistance prediction in Pseudomonas aeruginosa","author":"J Jeukens","year":"2017","journal-title":"Ann N Y Acad Sci"},{"issue":"7","key":"pcbi.1007349.ref028","doi-asserted-by":"crossref","first-page":"758","DOI":"10.1016\/S1473-3099(18)30225-1","article-title":"Public health surveillance of multidrug-resistant clones of Neisseria gonorrhoeae in Europe: a genomic survey","volume":"18","author":"SR Harris","year":"2018","journal-title":"Lancet Infect Dis"},{"issue":"8","key":"pcbi.1007349.ref029","article-title":"Genomic surveillance of Neisseria gonorrhoeae to investigate the distribution and evolution of antimicrobial-resistance determinants and lineages","volume":"4","author":"K Yahara","year":"2018","journal-title":"Microb Genom"},{"issue":"7","key":"pcbi.1007349.ref030","doi-asserted-by":"crossref","first-page":"1937","DOI":"10.1093\/jac\/dkx067","article-title":"WGS to predict antibiotic MICs for Neisseria gonorrhoeae","volume":"72","author":"DW Eyre","year":"2017","journal-title":"J Antimicrob Chemother"},{"issue":"4","key":"pcbi.1007349.ref031","doi-asserted-by":"crossref","DOI":"10.1128\/JCM.01934-17","article-title":"CLSI Methods Development and Standardization Working Group Best Practices for Evaluation of Antimicrobial Susceptibility Tests","volume":"56","author":"RM Humphries","year":"2018","journal-title":"J Clin Microbiol"},{"key":"pcbi.1007349.ref032","first-page":"jiy551","article-title":"Azithromycin susceptibility in Neisseria gonorrhoeae and seasonal macrolide use","author":"SW Olesen","year":"2018","journal-title":"J Infect Dis"},{"key":"pcbi.1007349.ref033","doi-asserted-by":"crossref","first-page":"E19","DOI":"10.1111\/j.1749-6632.2011.06215.x","article-title":"Antibiotic resistance in Neisseria gonorrhoeae: origin, evolution, and lessons learned for the future","volume":"1230","author":"M Unemo","year":"2011","journal-title":"Ann N Y Acad Sci"},{"issue":"4","key":"pcbi.1007349.ref034","doi-asserted-by":"crossref","first-page":"267 e5","DOI":"10.1016\/j.cmi.2016.12.009","article-title":"Novel rapid PCR for the detection of Ile491Phe rpoB mutation of Mycobacterium tuberculosis, a rifampicin-resistance-conferring mutation undetected by commercial assays","volume":"23","author":"E Andre","year":"2017","journal-title":"Clin Microbiol Infect"},{"key":"pcbi.1007349.ref035","doi-asserted-by":"crossref","first-page":"12797","DOI":"10.1038\/ncomms12797","article-title":"Sequence element enrichment analysis to determine the genetic basis of bacterial phenotypes","volume":"7","author":"JA Lees","year":"2016","journal-title":"Nat Commun"},{"issue":"7","key":"pcbi.1007349.ref036","doi-asserted-by":"crossref","first-page":"821","DOI":"10.1038\/ng.2310","article-title":"Genome-wide efficient mixed-model analysis for association studies","volume":"44","author":"X Zhou","year":"2012","journal-title":"Nat Genet"},{"issue":"10","key":"pcbi.1007349.ref037","doi-asserted-by":"crossref","first-page":"1183","DOI":"10.1038\/ng.2747","article-title":"Genomic analysis identifies targets of convergent positive selection in drug-resistant Mycobacterium tuberculosis","volume":"45","author":"MR Farhat","year":"2013","journal-title":"Nat Genet"},{"issue":"10","key":"pcbi.1007349.ref038","doi-asserted-by":"crossref","first-page":"1223","DOI":"10.1038\/nm.3937","article-title":"The association between sterilizing activity and drug distribution into tuberculosis lesions","volume":"21","author":"B Prideaux","year":"2015","journal-title":"Nat Med"},{"issue":"9","key":"pcbi.1007349.ref039","doi-asserted-by":"crossref","first-page":"965","DOI":"10.1097\/INF.0b013e31829043b3","article-title":"Outcomes of children with enterobacteriaceae bacteremia with reduced susceptibility to ceftriaxone: do the revised breakpoints translate to improved patient outcomes?","volume":"32","author":"PD Tamma","year":"2013","journal-title":"Pediatr Infect Dis J"},{"issue":"12","key":"pcbi.1007349.ref040","doi-asserted-by":"crossref","first-page":"4390","DOI":"10.1128\/AAC.01487-06","article-title":"Failure of current cefepime breakpoints to predict clinical outcomes of bacteremia caused by gram-negative organisms","volume":"51","author":"SV Bhat","year":"2007","journal-title":"Antimicrob Agents Chemother"},{"issue":"6","key":"pcbi.1007349.ref041","doi-asserted-by":"crossref","first-page":"862","DOI":"10.1086\/528712","article-title":"Outcomes of bacteremia due to Pseudomonas aeruginosa with reduced susceptibility to piperacillin-tazobactam: implications on the appropriateness of the resistance breakpoint","volume":"46","author":"VH Tam","year":"2008","journal-title":"Clin Infect Dis"},{"issue":"9","key":"pcbi.1007349.ref042","doi-asserted-by":"crossref","first-page":"823","DOI":"10.1056\/NEJMoa1715849","article-title":"Bacterial Factors That Predict Relapse after Tuberculosis Therapy","volume":"379","author":"R Colangeli","year":"2018","journal-title":"N Engl J Med"},{"issue":"5","key":"pcbi.1007349.ref043","doi-asserted-by":"crossref","first-page":"455","DOI":"10.1089\/cmb.2012.0021","article-title":"SPAdes: a new genome assembly algorithm and its applications to single-cell sequencing","volume":"19","author":"A Bankevich","year":"2012","journal-title":"J Comput Biol"},{"issue":"8","key":"pcbi.1007349.ref044","doi-asserted-by":"crossref","first-page":"1072","DOI":"10.1093\/bioinformatics\/btt086","article-title":"QUAST: quality assessment tool for genome assemblies","volume":"29","author":"A Gurevich","year":"2013","journal-title":"Bioinformatics"},{"issue":"3","key":"pcbi.1007349.ref045","doi-asserted-by":"crossref","first-page":"403","DOI":"10.1016\/S0022-2836(05)80360-2","article-title":"Basic local alignment search tool","volume":"215","author":"SF Altschul","year":"1990","journal-title":"J Mol Biol"},{"issue":"5","key":"pcbi.1007349.ref046","doi-asserted-by":"crossref","first-page":"1792","DOI":"10.1093\/nar\/gkh340","article-title":"MUSCLE: multiple sequence alignment with high accuracy and high throughput","volume":"32","author":"RC Edgar","year":"2004","journal-title":"Nucleic Acids Res"},{"key":"pcbi.1007349.ref047","article-title":"Aligning sequence reads, clone sequences and assembly contigs with BWA-MEM","author":"H. Li","year":"2013","journal-title":"arXiv e-prints"},{"issue":"11","key":"pcbi.1007349.ref048","doi-asserted-by":"crossref","first-page":"e112963","DOI":"10.1371\/journal.pone.0112963","article-title":"Pilon: an integrated tool for comprehensive microbial variant detection and genome assembly improvement","volume":"9","author":"BJ Walker","year":"2014","journal-title":"PLoS One"},{"key":"pcbi.1007349.ref049","doi-asserted-by":"crossref","first-page":"1","DOI":"10.18637\/jss.v077.i01","article-title":"ranger: A Fast Implementation of Random Forests for High Dimensional Data in C++ and R","volume":"77","author":"MN Wright","year":"2017","journal-title":"Journal of Statistical Software"},{"issue":"5","key":"pcbi.1007349.ref050","doi-asserted-by":"crossref","first-page":"652","DOI":"10.1093\/bioinformatics\/btt020","article-title":"DSK: k-mer counting with very low memory usage","volume":"29","author":"G Rizk","year":"2013","journal-title":"Bioinformatics"},{"key":"pcbi.1007349.ref051","doi-asserted-by":"crossref","first-page":"16041","DOI":"10.1038\/nmicrobiol.2016.41","article-title":"Identifying lineage effects when controlling for population structure improves power in bacterial association studies","volume":"1","author":"SG Earle","year":"2016","journal-title":"Nat Microbiol"},{"key":"pcbi.1007349.ref052","author":"The European Committee on Antimicrobial Susceptibility Testing","year":"2018","journal-title":"Breakpoint tables for interpretation of MICs and zone diameters"},{"key":"pcbi.1007349.ref053","author":"Clinical and Laboratory Standards Institute","year":"2019","journal-title":"CLSI M100: Performance Standards for Antimicrobial Susceptibility Testing, 29th Edition"},{"issue":"10","key":"pcbi.1007349.ref054","first-page":"27","article-title":"Evaluation Measures for Models Assessment over Imbalanced Data Sets","volume":"3","author":"M Bekkar","year":"2013","journal-title":"Journal of Information Engineering and Applications"},{"issue":"11","key":"pcbi.1007349.ref055","doi-asserted-by":"crossref","first-page":"1295","DOI":"10.1016\/S1473-3099(16)30157-8","article-title":"Whole-genome sequencing to determine transmission of Neisseria gonorrhoeae: an observational study","volume":"16","author":"D De Silva","year":"2016","journal-title":"Lancet Infect Dis"},{"issue":"1","key":"pcbi.1007349.ref056","doi-asserted-by":"crossref","first-page":"191","DOI":"10.1128\/JCM.02589-14","article-title":"Whole-genome phylogenomic heterogeneity of Neisseria gonorrhoeae isolates with decreased cephalosporin susceptibility collected in Canada between 1989 and 2013","volume":"53","author":"W Demczuk","year":"2015","journal-title":"J Clin Microbiol"},{"issue":"5","key":"pcbi.1007349.ref057","doi-asserted-by":"crossref","first-page":"1304","DOI":"10.1128\/JCM.03195-15","article-title":"Genomic Epidemiology and Molecular Resistance Mechanisms of Azithromycin-Resistant Neisseria gonorrhoeae in Canada from 1997 to 2014","volume":"54","author":"W Demczuk","year":"2016","journal-title":"J Clin Microbiol"},{"issue":"3","key":"pcbi.1007349.ref058","doi-asserted-by":"crossref","first-page":"220","DOI":"10.1016\/S1473-3099(13)70693-5","article-title":"Genomic epidemiology of Neisseria gonorrhoeae with reduced susceptibility to cefixime in the USA: a retrospective observational study","volume":"14","author":"YH Grad","year":"2014","journal-title":"Lancet Infect Dis"},{"issue":"2","key":"pcbi.1007349.ref059","doi-asserted-by":"crossref","first-page":"353","DOI":"10.1093\/jac\/dkx405","article-title":"Genomic epidemiology and antimicrobial resistance of Neisseria gonorrhoeae in New Zealand","volume":"73","author":"RS Lee","year":"2018","journal-title":"J Antimicrob Chemother"},{"issue":"3","key":"pcbi.1007349.ref060","doi-asserted-by":"crossref","first-page":"390","DOI":"10.1093\/cid\/ciu319","article-title":"The antimicrobial resistance monitoring and research (ARMoR) program: the US Department of Defense response to escalating antimicrobial resistance","volume":"59","author":"EP Lesho","year":"2014","journal-title":"Clin Infect Dis"}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1007349","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,9,13]],"date-time":"2019-09-13T00:00:00Z","timestamp":1568332800000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1007349","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,8]],"date-time":"2020-05-08T21:16:19Z","timestamp":1588972579000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1007349"}},"subtitle":[],"editor":[{"given":"Thomas R.","family":"Ioerger","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2019,9,3]]},"references-count":60,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2019,9,3]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1007349","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/607127","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,9,3]]}}}