{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,6]],"date-time":"2026-03-06T00:29:09Z","timestamp":1772756949764,"version":"3.50.1"},"reference-count":35,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,9,28]],"date-time":"2022-09-28T00:00:00Z","timestamp":1664323200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,9,28]],"date-time":"2022-09-28T00:00:00Z","timestamp":1664323200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Digit Imaging"],"DOI":"10.1007\/s10278-022-00706-8","type":"journal-article","created":{"date-parts":[[2022,9,28]],"date-time":"2022-09-28T19:02:37Z","timestamp":1664391757000},"page":"365-372","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":20,"title":["The 2021 SIIM-FISABIO-RSNA Machine Learning COVID-19 Challenge: Annotation and Standard Exam Classification of COVID-19 Chest Radiographs"],"prefix":"10.1007","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3373-9226","authenticated-orcid":false,"given":"Paras","family":"Lakhani","sequence":"first","affiliation":[]},{"given":"J.","family":"Mongan","sequence":"additional","affiliation":[]},{"given":"C.","family":"Singhal","sequence":"additional","affiliation":[]},{"given":"Q.","family":"Zhou","sequence":"additional","affiliation":[]},{"given":"K. P.","family":"Andriole","sequence":"additional","affiliation":[]},{"given":"W. F.","family":"Auffermann","sequence":"additional","affiliation":[]},{"given":"P. M.","family":"Prasanna","sequence":"additional","affiliation":[]},{"given":"T. X.","family":"Pham","sequence":"additional","affiliation":[]},{"given":"Michael","family":"Peterson","sequence":"additional","affiliation":[]},{"given":"P. J.","family":"Bergquist","sequence":"additional","affiliation":[]},{"given":"T. S.","family":"Cook","sequence":"additional","affiliation":[]},{"given":"S. F.","family":"Ferraciolli","sequence":"additional","affiliation":[]},{"given":"G. C. A.","family":"Corradi","sequence":"additional","affiliation":[]},{"given":"MS","family":"Takahashi","sequence":"additional","affiliation":[]},{"given":"C. S.","family":"Workman","sequence":"additional","affiliation":[]},{"given":"M.","family":"Parekh","sequence":"additional","affiliation":[]},{"given":"S. I.","family":"Kamel","sequence":"additional","affiliation":[]},{"given":"J.","family":"Galant","sequence":"additional","affiliation":[]},{"given":"A.","family":"Mas-Sanchez","sequence":"additional","affiliation":[]},{"given":"E. C.","family":"Ben\u00edtez","sequence":"additional","affiliation":[]},{"given":"M.","family":"S\u00e1nchez-Valverde","sequence":"additional","affiliation":[]},{"given":"L.","family":"Jaques","sequence":"additional","affiliation":[]},{"given":"M.","family":"Panadero","sequence":"additional","affiliation":[]},{"given":"M.","family":"Vidal","sequence":"additional","affiliation":[]},{"given":"M.","family":"Culia\u00f1ez-Casas","sequence":"additional","affiliation":[]},{"given":"D.","family":"Angulo-Gonzalez","sequence":"additional","affiliation":[]},{"given":"S. G.","family":"Langer","sequence":"additional","affiliation":[]},{"given":"Mar\u00eda","family":"de la Iglesia-Vay\u00e1","sequence":"additional","affiliation":[]},{"given":"G.","family":"Shih","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,9,28]]},"reference":[{"key":"706_CR1","doi-asserted-by":"crossref","unstructured":"Zhu N, Zhang D, Wang W, et al. A novel coronavirus from patients with pneumonia in China, 2019. New England Journal of Medicine. 2020 Jan 24.","DOI":"10.1056\/NEJMoa2001017"},{"key":"706_CR2","doi-asserted-by":"crossref","unstructured":"Liu Y, Gayle AA, Wilder-Smith A, et al. The reproductive number of COVID-19 is higher compared to SARS coronavirus. Journal of Travel Medicine. 2020 Mar 13","DOI":"10.1093\/jtm\/taaa021"},{"issue":"3","key":"706_CR3","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1016\/S2213-2600(20)30527-0","volume":"9","author":"L Piroth","year":"2021","unstructured":"Piroth L, Cottenet J, Mariet AS, et al. Comparison of the characteristics, morbidity, and mortality of COVID-19 and seasonal influenza: a nationwide, population-based retrospective cohort study. The Lancet Respiratory Medicine. 2021 Mar 1;9(3):251-9.","journal-title":"The Lancet Respiratory Medicine."},{"key":"706_CR4","unstructured":"World Health Organization, Coronavirus Dashboard. https:\/\/covid19.who.int\/ Accessed on 9\/15\/2021."},{"issue":"97","key":"706_CR5","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1016\/j.jped.2020.08.001","volume":"22","author":"ES Goudouris","year":"2021","unstructured":"Goudouris ES. Laboratory diagnosis of COVID-19. Jornal de Pediatria. 2021 Feb 22;97:7-12.","journal-title":"Jornal de Pediatria."},{"issue":"2","key":"706_CR6","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1177\/074823378500100206","volume":"1","author":"LW Hedlund","year":"1985","unstructured":"Hedlund LW, Putman CE. Methods for detecting pulmonary edema. Toxicology and Industrial Health. 1985 Apr;1(2):59-68.","journal-title":"Toxicology and Industrial Health."},{"issue":"10","key":"706_CR7","doi-asserted-by":"publisher","first-page":"1573","DOI":"10.1007\/s00134-012-2682-1","volume":"38","author":"ND Ferguson","year":"2012","unstructured":"Ferguson ND, Fan E, Camporota L, et al. The Berlin definition of ARDS: an expanded rationale, justification, and supplementary material. Intensive Care Medicine. 2012 Oct;38(10):1573-82.","journal-title":"Intensive Care Medicine."},{"key":"706_CR8","doi-asserted-by":"crossref","unstructured":"Mandell LA, Wunderink RG, Anzueto A, et al. Infectious Diseases Society of America\/American Thoracic Society consensus guidelines on the management of community-acquired pneumonia in adults. Clinical Infectious Diseases. 2007 Mar 1;44(Supplement_2):S27\u201372.","DOI":"10.1086\/511159"},{"issue":"1","key":"706_CR9","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1148\/radiol.2020201365","volume":"296","author":"GD Rubin","year":"2020","unstructured":"Rubin GD, Ryerson CJ, Haramati LB, et al. The role of chest imaging in patient management during the COVID-19 pandemic: a multinational consensus statement from the Fleischner Society. Radiology. 2020 Jul;296(1):172-80.","journal-title":"Radiology."},{"issue":"2","key":"706_CR10","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1148\/radiol.2017162326","volume":"284","author":"P Lakhani","year":"2017","unstructured":"Lakhani P, Sundaram B. Deep learning at chest radiography: automated classification of pulmonary tuberculosis by using convolutional neural networks. Radiology. 2017 Aug;284(2):574-82.","journal-title":"Radiology."},{"key":"706_CR11","unstructured":"Rajpurkar P, Irvin J, Zhu K, et al. Chexnet: Radiologist-level pneumonia detection on chest x-rays with deep learning. arXiv preprint arXiv:1711.05225. 2017 Nov 14."},{"issue":"11","key":"706_CR12","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1002697","volume":"15","author":"AG Taylor","year":"2018","unstructured":"Taylor AG, Mielke C, Mongan J. Automated detection of moderate and large pneumothorax on frontal chest X-rays using deep convolutional neural networks: A retrospective study. PLoS Medicine. 2018 Nov 20;15(11):e1002697.","journal-title":"PLoS Medicine."},{"key":"706_CR13","doi-asserted-by":"crossref","unstructured":"Ausawalaithong W, Thirach A, Marukatat S, et al. Automatic lung cancer prediction from chest X-ray images using the deep learning approach. In2018 11th Biomedical Engineering International Conference (BMEICON) 2018 Nov 21 (pp. 1\u20135). IEEE.","DOI":"10.1109\/BMEiCON.2018.8609997"},{"issue":"3","key":"706_CR14","doi-asserted-by":"publisher","first-page":"E166","DOI":"10.1148\/radiol.2020201874","volume":"296","author":"K Murphy","year":"2020","unstructured":"Murphy K, Smits H, Knoops AJ, et al. COVID-19 on chest radiographs: a multireader evaluation of an artificial intelligence system. Radiology. 2020 Sep;296(3):E166-72.","journal-title":"Radiology."},{"key":"706_CR15","doi-asserted-by":"crossref","unstructured":"Kundu S, Elhalawani H, Gichoya JW, et al. How might AI and chest imaging help unravel COVID-19\u2019s mysteries?. Radiology: Artificial Intelligence. 2020 May;2(3).","DOI":"10.1148\/ryai.2020200053"},{"key":"706_CR16","doi-asserted-by":"crossref","unstructured":"Li MD, Arun NT, Gidwani M, et al. Automated assessment and tracking of COVID-19 pulmonary disease severity on chest radiographs using convolutional siamese neural networks. Radiology: Artificial Intelligence. 2020 Jul 22;2(4):e200079.","DOI":"10.1148\/ryai.2020200079"},{"key":"706_CR17","doi-asserted-by":"crossref","unstructured":"Fridadar M, Amer R, Gozes O, et al. COVID-19 in CXR: From detection and severity scoring to patient disease monitoring. IEEE Journal of Biomedical and Health Informatics. 2021 Mar 26.","DOI":"10.1109\/JBHI.2021.3069169"},{"issue":"3","key":"706_CR18","doi-asserted-by":"publisher","first-page":"1770","DOI":"10.1007\/s00330-020-07269-8","volume":"31","author":"J Mushtaq","year":"2021","unstructured":"Mushtaq J, Pennella R, Lavalle S et al. Initial chest radiographs and artificial intelligence (AI) predict clinical outcomes in COVID-19 patients: analysis of 697 Italian patients. European Radiology. 2021 Mar;31(3):1770-9.","journal-title":"European Radiology."},{"issue":"3","key":"706_CR19","doi-asserted-by":"publisher","first-page":"623","DOI":"10.2214\/AJR.20.24801","volume":"217","author":"RA Reeves","year":"2021","unstructured":"Reeves RA, Pomeranz C, Gomella AA, et al. Performance of a severity score on admission chest radiography in predicting clinical outcomes in hospitalized patients with coronavirus disease (COVID-19). American Journal of Roentgenology. 2021;217(3):623-632.","journal-title":"American Journal of Roentgenology."},{"issue":"5","key":"706_CR20","doi-asserted-by":"publisher","first-page":"461","DOI":"10.1007\/s11547-020-01202-1","volume":"125","author":"A Borghesi","year":"2020","unstructured":"Borghesi A, Zigliani A, Masciullo R, et al. Radiographic severity index in COVID-19 pneumonia: relationship to age and sex in 783 Italian patients. La Radiologia Medica. 2020 May;125(5):461-4.","journal-title":"La Radiologia Medica."},{"issue":"4","key":"706_CR21","doi-asserted-by":"publisher","first-page":"572","DOI":"10.1016\/j.acra.2021.01.016","volume":"28","author":"MD Li","year":"2021","unstructured":"Li MD, Little BP, Alkasab TK, et al. Multi-radiologist user study for artificial intelligence-guided grading of COVID-19 lung disease severity on chest radiographs. Academic Radiology. 2021 Apr 1;28(4):572-6.","journal-title":"Academic Radiology."},{"key":"706_CR22","unstructured":"Vay\u00e1 MD, Saborit JM, Montell JA, et al. BIMCV COVID-19+: a large annotated dataset of RX and CT images from COVID-19 patients. arXiv preprint arXiv:2006.01174. 2020 Jun 1."},{"issue":"1","key":"706_CR23","doi-asserted-by":"publisher","first-page":"E204","DOI":"10.1148\/radiol.2021203957","volume":"299","author":"EB Tsai","year":"2021","unstructured":"Tsai EB, Simpson S, Lungren MP, et al. The RSNA International COVID-19 Open Radiology Database (RICORD). Radiology. 2021 Apr;299(1):E204-13.","journal-title":"Radiology."},{"issue":"2","key":"706_CR24","doi-asserted-by":"publisher","first-page":"498","DOI":"10.1148\/radiol.2018180736","volume":"290","author":"SS Halabi","year":"2019","unstructured":"Halabi SS, Prevedello LM, Kalpathy-Cramer J et al. The RSNA pediatric bone age machine learning challenge. Radiology. 2019 Feb;290(2):498-503.","journal-title":"Radiology."},{"key":"706_CR25","doi-asserted-by":"publisher","first-page":"490","DOI":"10.1007\/s10278-019-00299-9","volume":"33","author":"RW Filice","year":"2020","unstructured":"Filice, RW, Stein, A, Wu, CC, et al. Crowdsourcing pneumothorax annotations using machine learning annotations on the NIH chest X-ray dataset. J Digit Imaging 33, 490\u2013496 (2020). https:\/\/doi.org\/10.1007\/s10278-019-00299-9","journal-title":"J Digit Imaging"},{"key":"706_CR26","doi-asserted-by":"crossref","unstructured":"Flanders AE, Prevedello LM, Shih G, et al. Construction of a machine learning dataset through collaboration: the RSNA 2019 brain CT hemorrhage challenge. Radiology: Artificial Intelligence. 2020 Apr 29;2(3):e190211.","DOI":"10.1148\/ryai.2020190211"},{"key":"706_CR27","unstructured":"Colak E, Kitamura FC, Hobbs SB, et al. The RSNA Pulmonary Embolism CT Dataset. Radiology: Artificial Intelligence. 2021 Jan 20;3(2):e200254."},{"key":"706_CR28","doi-asserted-by":"crossref","unstructured":"Prevedello LM, Halabi SS, Shih G, et al. Challenges related to artificial intelligence research in medical imaging and the importance of image analysis competitions. Radiology: Artificial Intelligence. 2019 Jan 30;1(1):e180031.","DOI":"10.1148\/ryai.2019180031"},{"issue":"6","key":"706_CR29","doi-asserted-by":"crossref","first-page":"354","DOI":"10.1097\/RTI.0000000000000541","volume":"35","author":"DE Litmanovich","year":"2020","unstructured":"Litmanovich DE, Chung M, Kirkbride RR, et al. Review of chest radiograph findings of COVID-19 pneumonia and suggested reporting language. Journal of Thoracic Imaging. 2020 Nov 14;35(6):354-60.","journal-title":"Journal of Thoracic Imaging."},{"key":"706_CR30","unstructured":"Gamer M, Lemon J, Fellows I et al. IRR: Various coefficients of interrater reliability and agreement. R package version 0.84.1. https:\/\/CRAN.R-project.org\/package=irr"},{"issue":"4","key":"706_CR31","doi-asserted-by":"publisher","first-page":"284","DOI":"10.1037\/1040-3590.6.4.284","volume":"6","author":"DV Cicchetti","year":"1994","unstructured":"Cicchetti DV. Guidelines, criteria, and rules of thumb for evaluating normed and standardized assessment instruments in psychology. Psychol Assess. 1994;6(4):284-290.","journal-title":"Psychol Assess."},{"key":"706_CR32","unstructured":"American College of Radiology. ACR recommendations for the use of chest radiography and computed tomography (CT) for suspected COVID-19 infection. Available at: www.acr.org\/Advocacy-and-Economics\/ACR-Position-Statements\/Recommendations-for-Chest-Radiography-and-CT-for-Suspected-COVID19-Infection. Accessed August 3, 2021."},{"key":"706_CR33","first-page":"1","volume":"31","author":"AJ DeGrave","year":"2021","unstructured":"DeGrave AJ, Janizek JD, Lee SI. AI for radiographic COVID-19 detection selects shortcuts over signal. Nature Machine Intelligence. 2021 May 31:1-0.","journal-title":"Nature Machine Intelligence."},{"issue":"2","key":"706_CR34","doi-asserted-by":"publisher","first-page":"E72","DOI":"10.1148\/radiol.2020201160","volume":"296","author":"HY Wong","year":"2020","unstructured":"Wong HY, Lam HY, Fong AH, et al. Frequency and distribution of chest radiographic findings in patients positive for COVID-19. Radiology. 2020 Aug;296(2):E72-8.","journal-title":"Radiology."},{"issue":"7","key":"706_CR35","first-page":"13","volume":"14","author":"MB Weinstock","year":"2020","unstructured":"Weinstock MB, Echenique AN, Russell JW, et al. Chest x-ray findings in 636 ambulatory patients with COVID-19 presenting to an urgent care center: a normal chest x-ray is no guarantee. J Urgent Care Med. 2020;14(7):13-8.","journal-title":"J Urgent Care Med."}],"container-title":["Journal of Digital Imaging"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-022-00706-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10278-022-00706-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-022-00706-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,27]],"date-time":"2023-11-27T23:14:47Z","timestamp":1701126887000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10278-022-00706-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,28]]},"references-count":35,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2023,2]]}},"alternative-id":["706"],"URL":"https:\/\/doi.org\/10.1007\/s10278-022-00706-8","relation":{},"ISSN":["1618-727X"],"issn-type":[{"value":"1618-727X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,9,28]]},"assertion":[{"value":"21 October 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 September 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 September 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 September 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"This was a retrospective study using non-institutional public HIPAA compliant de-identified imaging data.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics Approval"}},{"value":"N\/A.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to Participate"}},{"value":"N\/A.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for Publication"}},{"value":"No other relevant conflicts of interest.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing Interests"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}