{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T15:42:48Z","timestamp":1776094968326,"version":"3.50.1"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2020,8,19]],"date-time":"2020-08-19T00:00:00Z","timestamp":1597795200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,8,19]],"date-time":"2020-08-19T00:00:00Z","timestamp":1597795200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["T32 Training Grant, T32EB001631"],"award-info":[{"award-number":["T32 Training Grant, T32EB001631"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Digit Imaging"],"published-print":{"date-parts":[[2020,10]]},"DOI":"10.1007\/s10278-020-00379-1","type":"journal-article","created":{"date-parts":[[2020,8,19]],"date-time":"2020-08-19T15:03:55Z","timestamp":1597849435000},"page":"1194-1201","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Between Always and Never: Evaluating Uncertainty in Radiology Reports Using Natural Language Processing"],"prefix":"10.1007","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2315-7767","authenticated-orcid":false,"given":"Andrew L.","family":"Callen","sequence":"first","affiliation":[]},{"given":"Sara M.","family":"Dupont","sequence":"additional","affiliation":[]},{"given":"Adi","family":"Price","sequence":"additional","affiliation":[]},{"given":"Ben","family":"Laguna","sequence":"additional","affiliation":[]},{"given":"David","family":"McCoy","sequence":"additional","affiliation":[]},{"given":"Bao","family":"Do","sequence":"additional","affiliation":[]},{"given":"Jason","family":"Talbott","sequence":"additional","affiliation":[]},{"given":"Marc","family":"Kohli","sequence":"additional","affiliation":[]},{"given":"Jared","family":"Narvid","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,8,19]]},"reference":[{"issue":"12","key":"379_CR1","doi-asserted-by":"publisher","first-page":"1221","DOI":"10.1001\/jamasurg.2014.382","volume":"149","author":"AJ Schoenfeld","year":"2014","unstructured":"Schoenfeld AJ, Harris MB, Davis M. Clinical uncertainty at the intersection of advancing technology, evidence-based medicine, and health care policy. JAMA Surg. 2014;149(12):1221\u20132.","journal-title":"JAMA Surg."},{"issue":"4","key":"379_CR2","doi-asserted-by":"publisher","first-page":"398","DOI":"10.1038\/nn2062","volume":"11","author":"ML Platt","year":"2008","unstructured":"Platt ML, Huettel SA. Risky business: the neuroeconomics of decision making under uncertainty. Nat Neurosci. 2008;11(4):398\u2013403.","journal-title":"Nat Neurosci."},{"key":"379_CR3","first-page":"65","volume":"8","author":"G Saposnik","year":"2017","unstructured":"Saposnik G, Sempere AP, Prefasi D, Selchen D, Ruff CC, Maurino J, et al. Decision-making in Multiple Sclerosis: The Role of Aversion to Ambiguity for Therapeutic Inertia among Neurologists (DIScUTIR MS). Front Neurol. 2017;8:65.","journal-title":"Front Neurol."},{"issue":"7","key":"379_CR4","doi-asserted-by":"publisher","first-page":"745","DOI":"10.1016\/j.crad.2014.03.004","volume":"69","author":"AB Rosenkrantz","year":"2014","unstructured":"Rosenkrantz AB, Kiritsy M, Kim S. How \u201cconsistent\u201d is \u201cconsistent\u201d? A clinician-based assessment of the reliability of expressions used by radiologists to communicate diagnostic confidence. Clin Radiol. 2014;69(7):745\u20139.","journal-title":"Clin Radiol."},{"issue":"6","key":"379_CR5","doi-asserted-by":"publisher","first-page":"685","DOI":"10.1016\/S1076-6332(03)80089-2","volume":"10","author":"R Khorasani","year":"2003","unstructured":"Khorasani R, Bates DW, Teeger S, Rothschild JM, Adams DF, Seltzer SE. Is terminology used effectively to convey diagnostic certainty in radiology reports? Acad Radiol. 2003;10(6):685\u20138.","journal-title":"Acad Radiol."},{"issue":"2","key":"379_CR6","doi-asserted-by":"publisher","first-page":"234","DOI":"10.1007\/s11606-006-0036-9","volume":"22","author":"PA Carney","year":"2007","unstructured":"Carney PA, Yi JP, Abraham LA, Miglioretti DL, Aiello EJ, Gerrity MS, et al. Reactions to uncertainty and the accuracy of diagnostic mammography. J Gen Intern Med. 2007;22(2):234\u201341.","journal-title":"J Gen Intern Med."},{"issue":"3","key":"379_CR7","doi-asserted-by":"publisher","first-page":"825","DOI":"10.1148\/radiology.169.3.3187005","volume":"169","author":"NJ Clinger","year":"1988","unstructured":"Clinger NJ, Hunter TB, Hillman BJ. Radiology reporting: attitudes of referring physicians. Radiology. 1988;169(3):825\u20136.","journal-title":"Radiology."},{"issue":"1","key":"379_CR8","doi-asserted-by":"publisher","first-page":"2","DOI":"10.2214\/AJR.15.15895","volume":"207","author":"DM Panicek","year":"2016","unstructured":"Panicek DM, Hricak H. How Sure Are You, Doctor? A Standardized Lexicon to Describe the Radiologist\u2019s Level of Certainty. AJR Am J Roentgenol. 2016;207(1):2\u20133.","journal-title":"AJR Am J Roentgenol."},{"issue":"11","key":"379_CR9","doi-asserted-by":"publisher","first-page":"1015","DOI":"10.1016\/j.crad.2011.05.013","volume":"66","author":"A Wallis","year":"2011","unstructured":"Wallis A, McCoubrie P. The radiology report\u2014are we getting the message across? Clin Radiol. 2011;66(11):1015\u201322.","journal-title":"Clin Radiol."},{"issue":"9","key":"379_CR10","doi-asserted-by":"publisher","first-page":"709","DOI":"10.1016\/S1076-6332(96)80407-7","volume":"3","author":"JL Sobel","year":"1996","unstructured":"Sobel JL, Pearson ML, Gross K, Desmond KA, Harrison ER, Rubenstein LV, et al. Information content and clarity of radiologists\u2019 reports for chest radiography. Acad Radiol. 1996;3(9):709\u201317.","journal-title":"Acad Radiol."},{"issue":"1","key":"379_CR11","doi-asserted-by":"publisher","first-page":"253","DOI":"10.2214\/ajr.176.1.1760253a","volume":"176","author":"C Valls","year":"2001","unstructured":"Valls C. Pitfalls of the vague radiology report. AJR Am J Roentgenol. 2001;176(1):253\u20134.","journal-title":"AJR Am J Roentgenol."},{"issue":"20","key":"379_CR12","doi-asserted-by":"publisher","first-page":"1619","DOI":"10.1001\/jama.1994.03520200075039","volume":"272","author":"W Levinson","year":"1994","unstructured":"Levinson W. Physician-patient communication. A key to malpractice prevention. JAMA. 1994;272(20):1619\u201320.","journal-title":"JAMA."},{"issue":"1","key":"379_CR13","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1016\/j.jacr.2016.08.027","volume":"14","author":"JK Hoang","year":"2017","unstructured":"Hoang JK. Do not hedge when there is certainty. J Am Coll Radiol. 2017;14(1):5.","journal-title":"J Am Coll Radiol."},{"issue":"12","key":"379_CR14","doi-asserted-by":"publisher","first-page":"851","DOI":"10.1016\/j.jacr.2009.07.023","volume":"6","author":"ES Burnside","year":"2009","unstructured":"Burnside ES, Sickles EA, Bassett LW, Rubin DL, Lee CH, Ikeda DM, et al. The ACR BI-RADS experience: learning from history. J Am Coll Radiol. 2009;6(12):851\u201360.","journal-title":"J Am Coll Radiol."},{"issue":"5","key":"379_CR15","doi-asserted-by":"publisher","first-page":"587","DOI":"10.1016\/j.jacr.2017.01.046","volume":"14","author":"FN Tessler","year":"2017","unstructured":"Tessler FN, Middleton WD, Grant EG, Hoang JK, Berland LL, Teefey SA, et al. ACR Thyroid Imaging, Reporting and Data System (TI-RADS): White Paper of the ACR TI-RADS Committee. J Am Coll Radiol. 2017;14(5):587\u201395.","journal-title":"J Am Coll Radiol."},{"issue":"3","key":"379_CR16","doi-asserted-by":"publisher","first-page":"517","DOI":"10.2214\/AJR.07.2209","volume":"189","author":"L Berlin","year":"2007","unstructured":"Berlin L. Radiologic errors and malpractice: a blurry distinction. AJR Am J Roentgenol. 2007;189(3):517\u201322.","journal-title":"AJR Am J Roentgenol."},{"issue":"1","key":"379_CR17","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1148\/radiol.2241011118","volume":"224","author":"G Hripcsak","year":"2002","unstructured":"Hripcsak G, Austin JHM, Alderson PO, Friedman C. Use of natural language processing to translate clinical information from a database of 889,921 chest radiographic reports. Radiology. 2002;224(1):157\u201363.","journal-title":"Radiology."},{"issue":"7","key":"379_CR18","doi-asserted-by":"publisher","first-page":"2039","DOI":"10.1148\/rg.307105083","volume":"30","author":"BH Do","year":"2010","unstructured":"Do BH, Wu A, Biswal S, Kamaya A, Rubin DL. Informatics in Radiology: RADTF: A Semantic Search\u2013enabled, Natural Language Processor\u2013generated Radiology Teaching File. Radiographics. 2010;30(7):2039\u201348.","journal-title":"Radiographics."},{"issue":"2","key":"379_CR19","doi-asserted-by":"publisher","first-page":"162","DOI":"10.1016\/j.annemergmed.2013.02.001","volume":"62","author":"S Dutta","year":"2013","unstructured":"Dutta S, Long WJ, Brown DFM, Reisner AT. Automated detection using natural language processing of radiologists recommendations for additional imaging of incidental findings. Ann Emerg Med. 2013;62(2):162\u20139.","journal-title":"Ann Emerg Med."},{"issue":"3","key":"379_CR20","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1016\/j.jacr.2007.09.003","volume":"5","author":"PA Dang","year":"2008","unstructured":"Dang PA, Kalra MK, Blake MA, Schultz TJ, Stout M, Lemay PR, et al. Natural language processing using online analytic processing for assessing recommendations in radiology reports. J Am Coll Radiol. 2008;5(3):197\u2013204.","journal-title":"J Am Coll Radiol."},{"issue":"1","key":"379_CR21","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1186\/1472-6920-7-8","volume":"7","author":"N Dogra","year":"2007","unstructured":"Dogra N, Giordano J, France N. Cultural diversity teaching and issues of uncertainty: the findings of a qualitative study. BMC Med Educ. 2007;7(1):8.","journal-title":"BMC Med Educ."},{"issue":"2","key":"379_CR22","doi-asserted-by":"publisher","first-page":"234","DOI":"10.1007\/s10278-009-9250-4","volume":"24","author":"AS Wu","year":"2011","unstructured":"Wu AS, Do BH, Kim J, Rubin DL. Evaluation of negation and uncertainty detection and its impact on precision and recall in search. J Digit Imaging. 2011;24(2):234\u201342.","journal-title":"J Digit Imaging."},{"key":"379_CR23","first-page":"321","volume":"2012","author":"DA Hanauer","year":"2012","unstructured":"Hanauer DA, Liu Y, Mei Q, Manion FJ, Balis UJ, Zheng K. Hedging their mets: the use of uncertainty terms in clinical documents and its potential implications when sharing the documents with patients. AMIA Annu Symp Proc. 2012;2012:321\u201330.","journal-title":"AMIA Annu Symp Proc."},{"issue":"1","key":"379_CR24","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1007\/s11606-017-4164-1","volume":"33","author":"V Bhise","year":"2018","unstructured":"Bhise V, Rajan SS, Sittig DF, Morgan RO, Chaudhary P, Singh H. Defining and measuring diagnostic uncertainty in medicine: a systematic review. J Gen Intern Med. 2018;33(1):103\u201315.","journal-title":"J Gen Intern Med."},{"issue":"2","key":"379_CR25","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1515\/dx-2014-0069","volume":"2","author":"L Zwaan","year":"2015","unstructured":"Zwaan L, Singh H. The challenges in defining and measuring diagnostic error. Diagnosis (Berl). 2015;2(2):97\u2013103.","journal-title":"Diagnosis (Berl)."},{"issue":"6","key":"379_CR26","doi-asserted-by":"publisher","first-page":"418","DOI":"10.1001\/jamainternmed.2013.2777","volume":"173","author":"H Singh","year":"2013","unstructured":"Singh H, Giardina TD, Meyer AND, Forjuoh SN, Reis MD, Thomas EJ. Types and origins of diagnostic errors in primary care settings. JAMA Intern Med. 2013;173(6):418\u201325.","journal-title":"JAMA Intern Med."},{"issue":"1","key":"379_CR27","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1148\/radiol.2018172572","volume":"287","author":"JK Hoang","year":"2018","unstructured":"Hoang JK, Middleton WD, Farjat AE, Langer JE, Reading CC, Teefey SA, et al. Reduction in thyroid nodule biopsies and improved accuracy with American college of radiology thyroid imaging reporting and data system. Radiology. 2018;287(1):185\u201393.","journal-title":"Radiology."},{"issue":"1","key":"379_CR28","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1148\/radiol.2017170554","volume":"286","author":"A Tang","year":"2018","unstructured":"Tang A, Bashir MR, Corwin MT, Cruite I, Dietrich CF, Do RKG, et al. Evidence supporting LI-RADS major features for CT- and MR imaging-based diagnosis of hepatocellular carcinoma: a systematic review. Radiology. 2018;286(1):29\u201348.","journal-title":"Radiology."},{"issue":"1","key":"379_CR29","doi-asserted-by":"publisher","first-page":"228","DOI":"10.1148\/radiol.2017161659","volume":"284","author":"H MacMahon","year":"2017","unstructured":"MacMahon H, Naidich DP, Goo JM, Lee KS, Leung ANC, Mayo JR, et al. Guidelines for Management of Incidental Pulmonary Nodules Detected on CT Images: From the Fleischner Society 2017. Radiology. 2017;284(1):228\u201343.","journal-title":"Radiology."},{"issue":"2","key":"379_CR30","doi-asserted-by":"publisher","first-page":"329","DOI":"10.1148\/radiol.16142770","volume":"279","author":"E Pons","year":"2016","unstructured":"Pons E, Braun LMM, Hunink MGM, Kors JA. Natural language processing in radiology: a systematic review. Radiology. 2016;279(2):329\u201343.","journal-title":"Radiology."},{"issue":"12","key":"379_CR31","doi-asserted-by":"publisher","first-page":"872","DOI":"10.1016\/j.jacr.2011.09.010","volume":"8","author":"R Lacson","year":"2011","unstructured":"Lacson R, Khorasani R. Practical examples of natural language processing in radiology. J Am Coll Radiol. 2011;8(12):872\u20134.","journal-title":"J Am Coll Radiol."},{"issue":"2","key":"379_CR32","doi-asserted-by":"publisher","first-page":"547","DOI":"10.1148\/rg.352140046","volume":"35","author":"A Srinivasa Babu","year":"2015","unstructured":"Srinivasa Babu A, Brooks ML. The malpractice liability of radiology reports: minimizing the risk. Radiographics. 2015;35(2):547\u201354.","journal-title":"Radiographics."},{"issue":"2","key":"379_CR33","doi-asserted-by":"publisher","first-page":"323","DOI":"10.1148\/radiol.2341040049","volume":"234","author":"KJ Dreyer","year":"2005","unstructured":"Dreyer KJ, Kalra MK, Maher MM, Hurier AM, Asfaw BA, Schultz T, et al. Application of recently developed computer algorithm for automatic classification of unstructured radiology reports: validation study. Radiology. 2005;234(2):323\u20139.","journal-title":"Radiology."},{"issue":"3","key":"379_CR34","first-page":"46","volume":"24","author":"IM Abbass","year":"2015","unstructured":"Abbass IM, Krause TM, Virani SS, Swint JM, Chan W, Franzini L. Revisiting the economic efficiencies of observation units. Manag Care. 2015;24(3):46\u201352.","journal-title":"Manag Care."},{"issue":"8","key":"379_CR35","doi-asserted-by":"publisher","first-page":"1076","DOI":"10.1016\/j.ajem.2015.04.025","volume":"33","author":"AM Prabhakar","year":"2015","unstructured":"Prabhakar AM, Misono AS, Harvey HB, Yun BJ, Saini S, Oklu R. Imaging utilization from the ED: no difference between observation and admitted patients. Am J Emerg Med. 2015;33(8):1076\u20139.","journal-title":"Am J Emerg Med."},{"key":"379_CR36","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1186\/1472-6963-8-91","volume":"8","author":"E Molins","year":"2008","unstructured":"Molins E, Maci\u00e0 F, Ferrer F, Maristany M-T, Castells X. Association between radiologists\u2019 experience and accuracy in interpreting screening mammograms. BMC Health Serv Res. 2008;8:91.","journal-title":"BMC Health Serv Res."},{"issue":"3","key":"379_CR37","doi-asserted-by":"publisher","first-page":"632","DOI":"10.1148\/radiol.2533090070","volume":"253","author":"DL Miglioretti","year":"2009","unstructured":"Miglioretti DL, Gard CC, Carney PA, Onega TL, Buist DSM, Sickles EA, et al. When radiologists perform best: the learning curve in screening mammogram interpretation. Radiology. 2009;253(3):632\u201340.","journal-title":"Radiology."},{"issue":"1","key":"379_CR38","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1186\/s12911-016-0377-1","volume":"16","author":"G Saposnik","year":"2016","unstructured":"Saposnik G, Redelmeier D, Ruff CC, Tobler PN. Cognitive biases associated with medical decisions: a systematic review. BMC Med Inform Decis Mak. 2016;16(1):138.","journal-title":"BMC Med Inform Decis Mak."}],"container-title":["Journal of Digital Imaging"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-020-00379-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10278-020-00379-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-020-00379-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,8,19]],"date-time":"2021-08-19T00:01:50Z","timestamp":1629331310000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10278-020-00379-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,8,19]]},"references-count":38,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2020,10]]}},"alternative-id":["379"],"URL":"https:\/\/doi.org\/10.1007\/s10278-020-00379-1","relation":{},"ISSN":["0897-1889","1618-727X"],"issn-type":[{"value":"0897-1889","type":"print"},{"value":"1618-727X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,8,19]]},"assertion":[{"value":"12 December 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 June 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 July 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 August 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with Ethical Standards"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}}]}}