{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,19]],"date-time":"2026-04-19T09:00:20Z","timestamp":1776589220236,"version":"3.51.2"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T00:00:00Z","timestamp":1746662400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T00:00:00Z","timestamp":1746662400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-025-01522-4","type":"journal-article","created":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T00:24:52Z","timestamp":1746663892000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Weakly supervised language models for automated extraction of critical findings from radiology reports"],"prefix":"10.1038","volume":"8","author":[{"given":"Avisha","family":"Das","sequence":"first","affiliation":[]},{"given":"Ish A.","family":"Talati","sequence":"additional","affiliation":[]},{"given":"Juan Manuel Zambrano","family":"Chaves","sequence":"additional","affiliation":[]},{"given":"Daniel","family":"Rubin","sequence":"additional","affiliation":[]},{"given":"Imon","family":"Banerjee","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,5,8]]},"reference":[{"key":"1522_CR1","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1016\/j.acra.2016.01.004","volume":"23","author":"P Cronin","year":"2016","unstructured":"Cronin, P. & Rawson, J. V. Review of research reporting guidelines for radiology researchers. Acad. Radiol. 23, 537\u2013558 (2016).","journal-title":"Acad. Radiol."},{"key":"1522_CR2","doi-asserted-by":"publisher","first-page":"809","DOI":"10.2214\/ajr.178.4.1780809","volume":"178","author":"L Berlin","year":"2002","unstructured":"Berlin, L. Communicating findings of radiologic examinations: whither goest the radiologist\u2019s duty? Am. J. Roentgenol. 178, 809\u2013815 (2002).","journal-title":"Am. J. Roentgenol."},{"key":"1522_CR3","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1016\/j.jacr.2004.06.015","volume":"2","author":"CL Sistrom","year":"2005","unstructured":"Sistrom, C. L. & Langlotz, C. P. A framework for improving radiology reporting. J. Am. Coll. Radiol. 2, 159\u2013167 (2005).","journal-title":"J. Am. Coll. Radiol."},{"key":"1522_CR4","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1007\/s10278-022-00712-w","volume":"36","author":"I Banerjee","year":"2023","unstructured":"Banerjee, I. et al. Natural language processing model for identifying critical findings-a multi-institutional study. J. Digital Imaging 36, 105\u2013113 (2023).","journal-title":"J. Digital Imaging"},{"key":"1522_CR5","doi-asserted-by":"publisher","first-page":"825","DOI":"10.1148\/radiology.169.3.3187005","volume":"169","author":"NJ Clinger","year":"1988","unstructured":"Clinger, N. J., Hunter, T. B. & Hillman, B. J. Radiology reporting: attitudes of referring physicians. Radiology 169, 825\u2013826 (1988).","journal-title":"Radiology"},{"key":"1522_CR6","doi-asserted-by":"crossref","unstructured":"Van Leeuwen, K. G., de Rooij, M., Schalekamp, S., van Ginneken, B. & Rutten, M. J. How does artificial intelligence in radiology improve efficiency and health outcomes? Pediatric Radiol. 52, 2087\u20132093 (2022).","DOI":"10.1007\/s00247-021-05114-8"},{"key":"1522_CR7","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1007\/s10278-011-9426-6","volume":"25","author":"P Lakhani","year":"2012","unstructured":"Lakhani, P., Kim, W. & Langlotz, C. P. Automated detection of critical results in radiology reports. J. Digital Imaging 25, 30\u201336 (2012).","journal-title":"J. Digital Imaging"},{"key":"1522_CR8","doi-asserted-by":"publisher","first-page":"1299","DOI":"10.1016\/j.jacr.2019.05.038","volume":"16","author":"ME Heilbrun","year":"2019","unstructured":"Heilbrun, M. E., Chapman, B. E., Narasimhan, E., Patel, N. & Mowery, D. Feasibility of natural language processing\u2013assisted auditing of critical findings in chest radiology. J. Am. Coll. Radiol. 16, 1299\u20131304 (2019).","journal-title":"J. Am. Coll. Radiol."},{"key":"1522_CR9","doi-asserted-by":"publisher","first-page":"988","DOI":"10.1007\/s10278-020-00349-7","volume":"33","author":"T Mabotuwana","year":"2020","unstructured":"Mabotuwana, T., Hall, C. S. & Cross, N. Framework for extracting critical findings in radiology reports. J. Digital Imaging 33, 988\u2013995 (2020).","journal-title":"J. Digital Imaging"},{"key":"1522_CR10","doi-asserted-by":"publisher","first-page":"3277","DOI":"10.1109\/TBME.2023.3280987","volume":"70","author":"Z Jiang","year":"2023","unstructured":"Jiang, Z. et al. Learning to summarize chinese radiology findings with a pre-trained encoder. IEEE Trans. Biomed. Eng. 70, 3277\u20133287 (2023).","journal-title":"IEEE Trans. Biomed. Eng."},{"key":"1522_CR11","unstructured":"Achiam, J. et al. GPT-4 technical report. https:\/\/arxiv.org\/abs\/2303.08774 (2023)."},{"key":"1522_CR12","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang, L. et al. Training language models to follow instructions with human feedback. Adv. neural Inf. Process. Syst. 35, 27730\u201327744 (2022).","journal-title":"Adv. neural Inf. Process. Syst."},{"key":"1522_CR13","unstructured":"Touvron, H. et al. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"1522_CR14","unstructured":"Jiang, A. Q. et al. Mistral 7b. Preprint at https:\/\/arxiv.org\/abs\/2302.13971 (2023)."},{"key":"1522_CR15","unstructured":"Wei, J. et al. Finetuned language models are zero-shot learners. Preprint at https:\/\/arxiv.org\/abs\/2109.01652 (2021)."},{"key":"1522_CR16","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T. et al. Language models are few-shot learners. Adv. Neural Inf. Process. Syst. 33, 1877\u20131901 (2020).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1522_CR17","unstructured":"Liu, L. et al. A survey on medical large language models: Technology, application, trustworthiness, and future directions. Preprint at https:\/\/arxiv.org\/abs\/2406.03712 (2024)."},{"key":"1522_CR18","first-page":"1","volume":"56","author":"B Wang","year":"2023","unstructured":"Wang, B. et al. Pre-trained language models in biomedical domain: A systematic survey. ACM Comput. Surv. 56, 1\u201352 (2023).","journal-title":"ACM Comput. Surv."},{"key":"1522_CR19","unstructured":"Wang, J., Yang, Z., Yao, Z. & Yu, H. JMLR: Joint medical llm and retrieval training for enhancing reasoning and professional question answering capability. Preprint at https:\/\/arxiv.org\/abs\/2402.17887 (2024)."},{"key":"1522_CR20","doi-asserted-by":"crossref","unstructured":"Woo, K.-m. C. et al. Evaluation of GPT-4 ability to identify and generate patient instructions for actionable incidental radiology findings. J. Am. Med. Informatics Assoc. 31, 1983\u20131993 (2024).","DOI":"10.1093\/jamia\/ocae117"},{"key":"1522_CR21","doi-asserted-by":"publisher","first-page":"e2330651","DOI":"10.2214\/AJR.23.30651","volume":"222","author":"R Bhayana","year":"2024","unstructured":"Bhayana, R. et al. Use of GPT-4 with single-shot learning to identify incidental findings in radiology reports. Am. J. Roentgenol. 222, e2330651 (2024).","journal-title":"Am. J. Roentgenol."},{"key":"1522_CR22","doi-asserted-by":"crossref","unstructured":"Kim, S. H. et al. Boosting LLM-assisted diagnosis: 10-minute llm tutorial elevates radiology residents\u2019 performance in brain mri interpretation. medRxiv, https:\/\/www.medrxiv.org\/content\/10.1101\/2024.07.03.24309779v1.full.pdf (2024).","DOI":"10.1101\/2024.07.03.24309779"},{"key":"1522_CR23","doi-asserted-by":"publisher","first-page":"552","DOI":"10.1016\/j.jacr.2013.12.016","volume":"11","author":"PA Larson","year":"2014","unstructured":"Larson, P. A., Berland, L. L., Griffith, B., Kahn Jr, C. E. & Liebscher, L. A. Actionable findings and the role of it support: report of the acr actionable reporting work group. J. Am. Coll. Radiol. 11, 552\u2013558 (2014).","journal-title":"J. Am. Coll. Radiol."},{"key":"1522_CR24","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2016.35","volume":"3","author":"AE Johnson","year":"2016","unstructured":"Johnson, A. E. et al. MIMIC-III, a freely accessible critical care database. Sci. Data 3, 1\u20139 (2016).","journal-title":"Sci. Data"},{"key":"1522_CR25","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-022-01899-x","volume":"10","author":"AE Johnson","year":"2023","unstructured":"Johnson, A. E. et al. MIMIC-IV, a freely accessible electronic health record dataset. Sci. Data 10, 1 (2023).","journal-title":"Sci. Data"},{"key":"1522_CR26","doi-asserted-by":"crossref","unstructured":"Labrak, Y. et al. Biomistral: A collection of open-source pretrained large language models for medical domains. Preprint at https:\/\/arxiv.org\/abs\/2402.10373 (2024).","DOI":"10.18653\/v1\/2024.findings-acl.348"},{"key":"1522_CR27","first-page":"74429","volume":"36","author":"J Zambrano Chaves","year":"2023","unstructured":"Zambrano Chaves, J. et al. Rales: a benchmark for radiology language evaluations. Adv. Neural Inf. Process. Syst. 36, 74429\u201374454 (2023).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1522_CR28","unstructured":"Mistral-7B huggingface model card. https:\/\/huggingface.co\/mistralai\/Mistral-7B-Instruct-v0.2 (2023). Accessed: 2024-09-30."},{"key":"1522_CR29","unstructured":"BioMistral-7B huggingface model card. https:\/\/huggingface.co\/BioMistral\/BioMistral-7B (2024). Accessed: 2024-09-30."},{"key":"1522_CR30","unstructured":"Gramopadhye, O. et al. Few shot chain-of-thought driven reasoning to prompt llms for open ended medical question answering. Findings of the Association for Computational Linguistics: EMNLP 2024 (2024)."},{"key":"1522_CR31","unstructured":"PyMedTermino toolkit. https:\/\/owlready2.readthedocs.io\/en\/latest\/pymedtermino2.html (2024). Accessed: 2024-09-30."},{"key":"1522_CR32","unstructured":"NCBO ontologies. https:\/\/bioportal.bioontology.org\/ (2011). Accessed: 2024-09-30."},{"key":"1522_CR33","unstructured":"FuzzyWuzzy python library. https:\/\/pypi.org\/project\/fuzzywuzzy\/ (2020). Accessed: 2024-09-30."},{"key":"1522_CR34","doi-asserted-by":"publisher","unstructured":"Wei, J. et al. Emergent abilities of large language models. Transact. Mach. Learn. Res. https:\/\/doi.org\/10.48550\/arXiv.2206.07682 (2022).","DOI":"10.48550\/arXiv.2206.07682"},{"key":"1522_CR35","unstructured":"Hu, E. J. et al. Lora: Low-rank adaptation of large language models. In Proceedings of The Tenth International Conference on Learning Representations (ICLR 2022)."},{"key":"1522_CR36","doi-asserted-by":"crossref","unstructured":"Papineni, K., Roukos, S., Ward, T. & Zhu, W.-J. Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting of the Association for Computational Linguistics 311\u2013318 (ACM, 2002).","DOI":"10.3115\/1073083.1073135"},{"key":"1522_CR37","unstructured":"Lin, C.-Y. Rouge: A package for automatic evaluation of summaries. Text summarization branches out 74\u201381 (Association for Computational Linguistics, 2004)."},{"key":"1522_CR38","unstructured":"Banerjee, S. & Lavie, A. METEOR: An automatic metric for mt evaluation with improved correlation with human judgments. In: Proceedings of the ACL workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization 65\u201372 (ACL, 2005)."},{"key":"1522_CR39","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K. Q. & Artzi, Y. Bertscore: Evaluating text generation with bert. Proceedings of International Conference on Learning Representations (2019)."},{"key":"1522_CR40","unstructured":"RadBERT model. https:\/\/github.com\/zzxslp\/RadBERT (2024). Accessed: 2024-09-30."},{"key":"1522_CR41","first-page":"e210258","volume":"4","author":"A Yan","year":"2022","unstructured":"Yan, A. et al. RadBERT: Adapting transformer-based language models to radiology. Radiology 4, e210258 (2022).","journal-title":"Radiology"},{"key":"1522_CR42","doi-asserted-by":"crossref","unstructured":"Liu, Y. et al. G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment. Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"1522_CR43","unstructured":"Kim, S. et al. Prometheus: Inducing fine-grained evaluation capability in language models. In: The Twelfth International Conference on Learning Representations (2023), OpenReview.net."},{"key":"1522_CR44","unstructured":"G-Eval model. https:\/\/github.com\/nlpyang\/geval (2023). Accessed: 2024-09-30."},{"key":"1522_CR45","first-page":"24824","volume":"35","author":"J Wei","year":"2022","unstructured":"Wei, J. et al. Chain-of-thought prompting elicits reasoning in large language models. Adv. Neural Inf. Process. Syst. 35, 24824\u201324837 (2022).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1522_CR46","unstructured":"Prometheus model. https:\/\/github.com\/prometheus-eval\/prometheus (2023). Accessed: 2024-09-30."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01522-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01522-4","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01522-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T13:04:43Z","timestamp":1746709483000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01522-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,8]]},"references-count":46,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2025,12]]}},"alternative-id":["1522"],"URL":"https:\/\/doi.org\/10.1038\/s41746-025-01522-4","relation":{"has-preprint":[{"id-type":"doi","id":"10.21203\/rs.3.rs-5060695\/v1","asserted-by":"object"}]},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,5,8]]},"assertion":[{"value":"10 September 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 February 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 May 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"257"}}