{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T08:32:50Z","timestamp":1775291570425,"version":"3.50.1"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1006269","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,4,16]],"date-time":"2019-04-16T00:00:00Z","timestamp":1555372800000}}],"reference-count":62,"publisher":"Public Library of Science (PLoS)","issue":"3","license":[{"start":{"date-parts":[[2019,3,27]],"date-time":"2019-03-27T00:00:00Z","timestamp":1553644800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100004318","name":"Microsoft","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100004318","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1006269","type":"journal-article","created":{"date-parts":[[2019,3,27]],"date-time":"2019-03-27T16:50:00Z","timestamp":1553705400000},"page":"e1006269","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":51,"title":["Evaluating reproducibility of AI algorithms in digital pathology with DAPPER"],"prefix":"10.1371","volume":"15","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1586-8350","authenticated-orcid":true,"given":"Andrea","family":"Bizzego","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2609-4354","authenticated-orcid":true,"given":"Nicole","family":"Bussola","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9791-9301","authenticated-orcid":true,"given":"Marco","family":"Chierici","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4824-893X","authenticated-orcid":true,"given":"Valerio","family":"Maggio","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1232-7302","authenticated-orcid":true,"given":"Margherita","family":"Francescatto","sequence":"additional","affiliation":[]},{"given":"Luca","family":"Cima","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0127-1342","authenticated-orcid":true,"given":"Marco","family":"Cristoforetti","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2705-5728","authenticated-orcid":true,"given":"Giuseppe","family":"Jurman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5384-3605","authenticated-orcid":true,"given":"Cesare","family":"Furlanello","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2019,3,27]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-319-42999-1","article-title":"Deep Learning and Convolutional Neural Networks for Medical Image Computing","author":"L Lu","year":"2017"},{"issue":"7639","key":"ref2","doi-asserted-by":"crossref","first-page":"115","DOI":"10.1038\/nature21056","article-title":"Dermatologist-level classification of skin cancer with deep neural networks","volume":"542","author":"A Esteva","year":"2017","journal-title":"Nature"},{"issue":"Suppl. 1","key":"ref3","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1159\/000481227","article-title":"Computer-Aided Diagnosis Based on Convolutional Neural Network System for Colorectal Polyp Classification: Preliminary Experience","volume":"93","author":"Y Komeda","year":"2017","journal-title":"Oncology"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"30","DOI":"10.4103\/jpi.jpi_34_17","article-title":"Deep Learning for Classification of Colorectal Polyps on Whole-Slide Images","volume":"8","author":"B Korbar","year":"2017","journal-title":"Journal of Pathology Informatics"},{"issue":"9","key":"ref5","doi-asserted-by":"crossref","first-page":"1342","DOI":"10.1038\/s41591-018-0107-6","article-title":"Clinically applicable deep learning for diagnosis and referral in retinal disease","volume":"24","author":"J De Fauw","year":"2018","journal-title":"Nature Medicine"},{"key":"ref6","doi-asserted-by":"crossref","first-page":"46479","DOI":"10.1038\/srep46479","article-title":"Towards automatic pulmonary nodule management in lung cancer screening with deep learning","volume":"7","author":"F Ciompi","year":"2017","journal-title":"Scientific Reports"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1016\/j.media.2017.07.005","article-title":"A survey on deep learning in medical image analysis","volume":"42","author":"G Litjens","year":"2017","journal-title":"Medical Image Analysis"},{"issue":"1","key":"ref8","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1038\/s41746-018-0048-y","article-title":"With an eye to AI and autonomous diagnosis","volume":"1","author":"PA Keane","year":"2018","journal-title":"NPJ Digital Medicine"},{"issue":"1","key":"ref9","doi-asserted-by":"crossref","first-page":"5217","DOI":"10.1038\/s41467-018-07619-7","article-title":"Why rankings of biomedical image analysis competitions should be interpreted with care","volume":"9","author":"L Maier-Hein","year":"2018","journal-title":"Nature Communications"},{"issue":"1","key":"ref10","doi-asserted-by":"crossref","first-page":"588","DOI":"10.1038\/s41467-019-08563-w","article-title":"Author Correction: Why rankings of biomedical image analysis competitions should be interpreted with care","volume":"10","author":"L Maier-Hein","year":"2019","journal-title":"Nature Communications"},{"issue":"13","key":"ref11","doi-asserted-by":"crossref","first-page":"E2970","DOI":"10.1073\/pnas.1717139115","article-title":"Predicting cancer outcomes from histology and genomics using convolutional networks","volume":"115","author":"P Mobadersany","year":"2018","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"1","key":"ref12","doi-asserted-by":"crossref","first-page":"3395","DOI":"10.1038\/s41598-018-21758-3","article-title":"Deep learning based tissue analysis predicts outcome in colorectal cancer","volume":"8","author":"D Bychkov","year":"2018","journal-title":"Scientific Reports"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1016\/j.compmedimag.2017.06.001","article-title":"Deep convolutional neural networks for automatic classification of gastric carcinoma using whole slide images in digital histopathology","volume":"61","author":"H Sharma","year":"2017","journal-title":"Computerized Medical Imaging and Graphics"},{"key":"ref14","doi-asserted-by":"crossref","unstructured":"Paeng K, Hwang S, Park S, Kim M. A unified framework for tumor proliferation score prediction in breast histopathology. In: Proceedings of the Third International Workshop on Deep Learning in Medical Image Analysis (DLMIA 2017) and the Sixth International Workshop on Multimodal Learning for Clinical Decision Support (ML-CDS 2017), held in conjunction with the Twentieth International Conference on Medical Imaging and Computer-Assisted Intervention (MICCAI 2017). Springer; 2017. p. 231\u2013239.","DOI":"10.1007\/978-3-319-67558-9_27"},{"issue":"10","key":"ref15","doi-asserted-by":"crossref","first-page":"1559","DOI":"10.1038\/s41591-018-0177-5","article-title":"Classification And Mutation Prediction From Non-Small Cell Lung Cancer Histopathology Images Using Deep Learning","volume":"24","author":"N Coudray","year":"2018","journal-title":"Nature Medicine"},{"issue":"Supp.13","key":"ref16","doi-asserted-by":"crossref","first-page":"5309","DOI":"10.1158\/1538-7445.AM2018-5309","article-title":"Determining EGFR and STK11 mutational status in lung adenocarcinoma histopathology images using deep learning","volume":"78","author":"N Coudray","year":"2018","journal-title":"Cancer Research"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"2089","DOI":"10.1109\/TBME.2013.2245129","article-title":"Multi-field-of-view framework for distinguishing tumor grade in ER+ breast cancer from entire histopathology slides","volume":"60","author":"A Basavanhally","year":"2013","journal-title":"IEEE Transactions on Biomedical Engineering"},{"issue":"10","key":"ref18","doi-asserted-by":"crossref","first-page":"1155","DOI":"10.1038\/modpathol.2016.109","article-title":"Standardized evaluation of tumor-infiltrating lymphocytes in breast cancer: results of the ring studies of the international immuno-oncology biomarker working group","volume":"29","author":"C Denkert","year":"2016","journal-title":"Modern Pathology"},{"issue":"9","key":"ref19","doi-asserted-by":"crossref","first-page":"e1019981","DOI":"10.1080\/2162402X.2015.1019981","article-title":"Tumor-infiltrating T lymphocytes improve clinical outcome of therapy-resistant neuroblastoma","volume":"4","author":"M Mina","year":"2015","journal-title":"Oncoimmunology"},{"issue":"1","key":"ref20","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1016\/S1470-2045(17)30905-1","article-title":"Tumour infiltrating lymphocytes in breast cancer: increasing clinical relevance","volume":"19","author":"R Salgado","year":"2018","journal-title":"The Lancet Oncology"},{"issue":"1","key":"ref21","doi-asserted-by":"crossref","first-page":"74","DOI":"10.1080\/0284186X.2017.1400180","article-title":"Triple negative breast cancer\u2013prognostic role of immune-related factors: a systematic review","volume":"57","author":"ES Stovgaard","year":"2018","journal-title":"Acta Oncologica"},{"issue":"1","key":"ref22","first-page":"151","article-title":"Tumor-infiltrating Lymphocytes Predict the Chemotherapeutic Outcomes in Patients with Stage IV Colorectal Cancer","volume":"32","author":"M Shibutani","year":"2018","journal-title":"In Vivo"},{"key":"ref23","doi-asserted-by":"crossref","unstructured":"Deng J, Dong W, Socher R, Li LJ, Li K, Li FF. Imagenet: A large-scale hierarchical image database. In: Proceedings of the 2009 IEEE Conference on Computer Vision and Pattern Recognition (CVPR). IEEE; 2009. p. 248\u2013255.","DOI":"10.1109\/CVPR.2009.5206848"},{"issue":"5","key":"ref24","doi-asserted-by":"crossref","first-page":"1299","DOI":"10.1109\/TMI.2016.2535302","article-title":"Convolutional neural networks for medical image analysis: Full training or fine tuning?","volume":"35","author":"N Tajbakhsh","year":"2016","journal-title":"IEEE Transactions on Medical Imaging"},{"key":"ref25","doi-asserted-by":"crossref","unstructured":"Kieffer B, Babaie M, Kalra S, Tizhoosh HR. Convolutional Neural Networks for Histopathology Image Classification: Training vs. Using Pre-Trained Networks. In: Proceedings of the Seventh International Conference on Image Processing Theory, Tools and Applications (IPTA 2017). IEEE; 2017. p. 1\u20136.","DOI":"10.1109\/IPTA.2017.8310149"},{"issue":"2","key":"ref26","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1038\/ng.295","article-title":"Repeatability of published microarray gene expression analyses","volume":"41","author":"JPA Ioannidis","year":"2009","journal-title":"Nature Genetics"},{"issue":"7604","key":"ref27","doi-asserted-by":"crossref","first-page":"452","DOI":"10.1038\/533452a","article-title":"1,500 scientists lift the lid on reproducibility","volume":"533","author":"M Baker","year":"2016","journal-title":"Nature News"},{"issue":"12","key":"ref28","doi-asserted-by":"crossref","first-page":"1127","DOI":"10.1038\/nbt.4029","article-title":"The international MAQC Society launches to enhance reproducibility of high-throughput technologies","volume":"35","author":"L Shi","year":"2017","journal-title":"Nature Biotechnology"},{"key":"ref29","doi-asserted-by":"crossref","first-page":"160018","DOI":"10.1038\/sdata.2016.18","article-title":"The FAIR Guiding Principles for scientific data management and stewardship","volume":"3","author":"MD Wilkinson","year":"2016","journal-title":"Scientific Data"},{"issue":"8","key":"ref30","doi-asserted-by":"crossref","first-page":"827","DOI":"10.1038\/nbt.1665","article-title":"The MAQC-II Project: A comprehensive study of common practices for the development and validation of microarray-based predictive models","volume":"28","year":"2010","journal-title":"Nature Biotechnology"},{"key":"ref31","doi-asserted-by":"crossref","first-page":"903","DOI":"10.1038\/nbt.2957","article-title":"A comprehensive assessment of RNA-seq accuracy, reproducibility and information content by the Sequence Quality Control consortium","volume":"32","year":"2014","journal-title":"Nature Biotechnology"},{"issue":"2","key":"ref32","doi-asserted-by":"crossref","first-page":"258","DOI":"10.1093\/bioinformatics\/btm550","article-title":"Algebraic stability indicators for ranked lists in molecular profiling","volume":"24","author":"G Jurman","year":"2008","journal-title":"Bioinformatics"},{"issue":"1","key":"ref33","doi-asserted-by":"crossref","first-page":"54","DOI":"10.1186\/1471-2105-4-54","article-title":"Entropy-based gene ranking without selection bias for the predictive classification of microarray data","volume":"4","author":"C Furlanello","year":"2003","journal-title":"BMC Bioinformatics"},{"key":"ref34","doi-asserted-by":"crossref","unstructured":"Lin TY, Maire M, Belongie S, Hays J, Perona P, Ramanan D, et al. Microsoft COCO: Common Objects in Context. In: Proceedings of the Thirteenth European Conference on Computer Vision (ECCV 2014). Springer; 2014. p. 740\u2013755.","DOI":"10.1007\/978-3-319-10602-1_48"},{"issue":"6","key":"ref35","doi-asserted-by":"crossref","first-page":"580","DOI":"10.1038\/ng.2653","article-title":"The genotype-tissue expression (GTEx) project","volume":"45","year":"2013","journal-title":"Nature Genetics"},{"issue":"3","key":"ref36","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1007\/BF00994018","article-title":"Support-vector networks","volume":"20","author":"C Cortes","year":"1995","journal-title":"Machine Learning"},{"key":"ref37","unstructured":"Ho TK. Random decision forests. In: Proceedings of 3rd International Conference on Document Analysis and Recognition (ICDAR 1995). IEEE; 1995. p. 278\u2013282)."},{"key":"ref38","doi-asserted-by":"crossref","unstructured":"Babaie M, Kalra S, Sriram A, Mitcheltree C, Zhu S, Khatami A, et al. Classification and Retrieval of Digital Pathology Scans: A New Dataset. In: Proceedings of the 2017 IEEE Conference on Computer Vision and Pattern Recognition Workshops (CVPRW). IEEE; 2017. p. 8\u201316.","DOI":"10.1109\/CVPRW.2017.106"},{"key":"ref39","doi-asserted-by":"crossref","unstructured":"Kumar MD, Babaie M, Zhu S, Kalra S, Tizhoosh HR. A Comparative Study of CNN, BoVW and LBP for Classification of Histopathological Images. In: Proceedings of the 2017 IEEE Symposium Series on Computational Intelligence (SSCI). IEEE; 2017. p. 1\u20137.","DOI":"10.1109\/SSCI.2017.8285162"},{"key":"ref40","doi-asserted-by":"crossref","unstructured":"Alhindi TJ, Kalra S, Ng KH, Afrin A, Tizhoosh HR. Comparing LBP, HOG and Deep Features for Classification of Histopathology Images. In: Proceedings of the 2018 International Joint Conference on Neural Networks (IJCNN). IEEE; 2018. p. 1\u20137.","DOI":"10.1109\/IJCNN.2018.8489329"},{"issue":"5","key":"ref41","doi-asserted-by":"crossref","first-page":"311","DOI":"10.1089\/bio.2015.0032","article-title":"A Novel Approach to High-Quality Postmortem Tissue Procurement: The GTEx Project","volume":"13","author":"LJ Carithers","year":"2015","journal-title":"Biopreservation and Biobanking"},{"key":"ref42","unstructured":"Nader Vasconcelos C, Nader Vasconcelos B. Increasing deep learning melanoma classification by classical and expert knowledge based image transforms. arXiv. 2017;1702.07025."},{"key":"ref43","doi-asserted-by":"crossref","unstructured":"Cubuk ED, Zoph B, Mane D, Vasudevan V, Le QV. AutoAugment: Learning Augmentation Policies from Data. arXiv. 2018;1805.09501.","DOI":"10.1109\/CVPR.2019.00020"},{"key":"ref44","unstructured":"Wang J, Perez L. The effectiveness of data augmentation in image classification using deep learning. arXiv. 2017;1712.04621."},{"key":"ref45","unstructured":"Simonyan K, Zisserman A. Very deep convolutional networks for large-scale image recognition. In: Proceedings of the Third International Conference on Learning Representations (ICLR 2015). arXiv:1409.1556; 2015. p. 1\u201314."},{"key":"ref46","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S, Sun J. Deep Residual Learning for Image Recognition. In: Proceedings of the 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR). IEEE; 2016. p. 770\u2013778.","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref47","doi-asserted-by":"crossref","unstructured":"Szegedy C, Vanhoucke V, Ioffe S, Shlens J, Wojna Z. Rethinking the Inception Architecture for Computer Vision. In: Proceedings of the 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR). IEEE; 2016. p. 2818\u20132826.","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref48","unstructured":"Canziani A, Paszke A, Culurciello E. An analysis of deep neural network models for practical applications. arXiv. 2017;1605.076784:1\u20137."},{"key":"ref49","unstructured":"Hinton GE, Srivastava N, Krizhevsky A, Sutskever I, Salakhutdinov RR. Improving neural networks by preventing co-adaptation of feature detectors. arXiv. 2012;1207.0580:1\u201318."},{"key":"ref50","unstructured":"Kinga D, Adam JB. Adam: A Method for Stochastic Optimization. In: Proceedings of the Third International Conference on Learning Representations (ICLR 2015). arXiv:1412.6980; 2014. p. 1\u201315."},{"key":"ref51","article-title":"The Elements of Statistical Learning: Data Mining, Inference, and Prediction","author":"T Hastie","year":"2009"},{"issue":"2","key":"ref52","doi-asserted-by":"crossref","first-page":"442","DOI":"10.1016\/0005-2795(75)90109-9","article-title":"Comparison of the predicted and observed secondary structure of T4 phage lysozyme","volume":"405","author":"BW Matthews","year":"1975","journal-title":"Biochimica et Biophysica Acta"},{"issue":"5","key":"ref53","doi-asserted-by":"crossref","first-page":"412","DOI":"10.1093\/bioinformatics\/16.5.412","article-title":"Assessing the accuracy of prediction algorithms for classification: an overview","volume":"16","author":"P Baldi","year":"2000","journal-title":"Bioinformatics"},{"issue":"8","key":"ref54","doi-asserted-by":"crossref","first-page":"e41882","DOI":"10.1371\/journal.pone.0041882","article-title":"A comparison of MCC and CEN error measures in multi-class prediction","volume":"7","author":"G Jurman","year":"2012","journal-title":"PLOS ONE"},{"key":"ref55","doi-asserted-by":"crossref","first-page":"35","DOI":"10.1186\/s13040-017-0155-3","article-title":"Ten quick tips for machine learning in computational biology","volume":"10","author":"D Chicco","year":"2017","journal-title":"BioData Mining"},{"key":"ref56","doi-asserted-by":"crossref","first-page":"189","DOI":"10.1214\/ss\/1032280214","article-title":"Bootstrap confidence intervals (with Discussion)","volume":"11","author":"TJ Di Ciccio","year":"1996","journal-title":"Statistical Science"},{"key":"ref57","first-page":"2579","article-title":"Visualizing data using t-SNE","volume":"9","author":"L Van der Maaten","year":"2008","journal-title":"Journal of Machine Learning Research"},{"issue":"29","key":"ref58","doi-asserted-by":"crossref","first-page":"861","DOI":"10.21105\/joss.00861","article-title":"UMAP: Uniform Manifold Approximation and Projection","volume":"3","author":"L McInnes","year":"2018","journal-title":"Journal of Open Source Software"},{"key":"ref59","article-title":"Dimensionality reduction for visualizing single-cell data using UMAP","author":"E Becht","year":"2018","journal-title":"Nature Biotechnology"},{"issue":"8","key":"ref60","doi-asserted-by":"crossref","first-page":"775","DOI":"10.1038\/nmeth.4326","article-title":"Image Data Resource: a bioimage data integration and publication platform","volume":"14","author":"E Williams","year":"2017","journal-title":"Nature Methods"},{"issue":"4","key":"ref61","doi-asserted-by":"crossref","first-page":"e0192726","DOI":"10.1371\/journal.pone.0192726","article-title":"A deep-learning classifier identifies patients with clinical heart failure using whole-slide images of H&E tissue","volume":"13","author":"JJ Nirschl","year":"2018","journal-title":"PLOS ONE"},{"issue":"10","key":"ref62","doi-asserted-by":"crossref","first-page":"R100","DOI":"10.1186\/gb-2006-7-10-r100","article-title":"CellProfiler: image analysis software for identifying and quantifying cell phenotypes","volume":"7","author":"AE Carpenter","year":"2006","journal-title":"Genome Biology"}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1006269","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,4,16]],"date-time":"2019-04-16T00:00:00Z","timestamp":1555372800000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1006269","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,12,3]],"date-time":"2020-12-03T21:37:46Z","timestamp":1607031466000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1006269"}},"subtitle":[],"editor":[{"given":"Gustavo","family":"Rohde","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2019,3,27]]},"references-count":62,"journal-issue":{"issue":"3","published-online":{"date-parts":[[2019,3,27]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1006269","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/340646","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,3,27]]}}}