{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T10:55:41Z","timestamp":1782298541111,"version":"3.54.5"},"reference-count":46,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T00:00:00Z","timestamp":1775520000000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100010135","name":"Finska L\u00e4kares\u00e4llskapet","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100010135","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100008723","name":"Suomen L\u00e4\u00e4ketieteen S\u00e4\u00e4ti\u00f6","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100008723","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100014438","name":"Business Finland","doi-asserted-by":"publisher","award":["1562\/31\/2024"],"award-info":[{"award-number":["1562\/31\/2024"]}],"id":[{"id":"10.13039\/501100014438","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010116","name":"Medicinska Underst\u00f6dsf\u00f6reningen Liv och H\u00e4lsa rf","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100010116","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["clinicalkey.com","clinicalkey.com.au","clinicalkey.es","clinicalkey.fr","clinicalkey.jp","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Artificial Intelligence in Medicine"],"published-print":{"date-parts":[[2026,7]]},"DOI":"10.1016\/j.artmed.2026.103421","type":"journal-article","created":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T16:24:51Z","timestamp":1775579091000},"page":"103421","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":2,"special_numbering":"C","title":["Measuring the quality of AI-generated clinical notes: A systematic review and experimental benchmark of evaluation methods"],"prefix":"10.1016","volume":"177","author":[{"given":"Alexandra","family":"Dahlberg","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tiila","family":"K\u00e4enniemi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tiia","family":"Winther-Jensen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Olli","family":"Tapiola","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rami","family":"Luisto","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tuukka","family":"Puranen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Max","family":"Gordon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Enni","family":"Sanmark","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ville","family":"Vartiainen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.artmed.2026.103421_bb0005","doi-asserted-by":"crossref","first-page":"36","DOI":"10.1186\/s12873-018-0188-z","article-title":"Strategies for improving physician documentation in the emergency department: a systematic review","volume":"18","author":"Lorenzetti","year":"2018","journal-title":"BMC Emerg Med"},{"key":"10.1016\/j.artmed.2026.103421_bb0010","doi-asserted-by":"crossref","first-page":"e169","DOI":"10.1093\/jamia\/ocv152","article-title":"Risks and benefits of speech recognition for clinical documentation: a systematic review","volume":"23","author":"Hodgson","year":"2016","journal-title":"J Am Med Inform Assoc"},{"key":"10.1016\/j.artmed.2026.103421_bb0015","doi-asserted-by":"crossref","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","article-title":"Large language models in medicine","volume":"29","author":"Thirunavukarasu","year":"2023","journal-title":"Nat Med"},{"key":"10.1016\/j.artmed.2026.103421_bb0020","doi-asserted-by":"crossref","DOI":"10.1038\/s44401-024-00011-2","article-title":"Current and future state of evaluation of large language models for medical summarization tasks","volume":"2","author":"Croxford","year":"2025","journal-title":"NPJ Health Syst"},{"key":"10.1016\/j.artmed.2026.103421_bb0025","article-title":"The PRISMA 2020 statement: an updated guideline for reporting systematic reviews","volume":"372","author":"Page","year":"2021","journal-title":"BMJ"},{"key":"10.1016\/j.artmed.2026.103421_bb0035","doi-asserted-by":"crossref","first-page":"261","DOI":"10.1038\/s41592-019-0686-2","article-title":"SciPy 1.0: fundamental algorithms for scientific computing in Python","volume":"17","author":"Virtanen","year":"2020","journal-title":"Nat Methods"},{"key":"10.1016\/j.artmed.2026.103421_bb0040","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1109\/MCSE.2007.55","article-title":"Matplotlib: a 2D graphics environment","volume":"9","author":"Hunter","year":"2007","journal-title":"Computing in Science & Engineering"},{"key":"10.1016\/j.artmed.2026.103421_bb0045","doi-asserted-by":"crossref","first-page":"357","DOI":"10.1038\/s41586-020-2649-2","article-title":"Array programming with NumPy","volume":"585","author":"Harris","year":"2020","journal-title":"Nature"},{"key":"10.1016\/j.artmed.2026.103421_bb0050","first-page":"2825","article-title":"Scikit-learn: machine learning in Python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"J Mach Learn Res"},{"key":"10.1016\/j.artmed.2026.103421_bb0055","series-title":"Proceedings of the Third Conference on Machine Translation: Research Papers","article-title":"A call for clarity in reporting {BLEU} scores","author":"Post","year":"2018"},{"key":"10.1016\/j.artmed.2026.103421_bb0060","doi-asserted-by":"crossref","DOI":"10.21105\/joss.03021","article-title":"Seaborn: statistical data visualization","volume":"6","author":"Waskom","year":"2021","journal-title":"J Open Source Softw"},{"key":"10.1016\/j.artmed.2026.103421_bb0065","author":"Google LLC"},{"key":"10.1016\/j.artmed.2026.103421_bb0070","series-title":"Proc 8th International Conference on Learning Representations (ICLR20)","article-title":"BERTScore: evaluating text generation with BERT","author":"Zhang","year":"2020"},{"key":"10.1016\/j.artmed.2026.103421_bb0075","series-title":"Proceedings of the 2020 conference on empirical methods in natural language processing: System Demonstrations","article-title":"Transformers: State-of-the-art natural language processing","author":"Wolf","year":"2020"},{"key":"10.1016\/j.artmed.2026.103421_bb0080","author":"Microsoft Corporation"},{"key":"10.1016\/j.artmed.2026.103421_bb0085","series-title":"ROUGE: A Package for Automatic Evaluation of Summaries","first-page":"74","author":"Lin","year":"2004"},{"key":"10.1016\/j.artmed.2026.103421_bb0090","series-title":"Bleu: a Method for Automatic Evaluation of Machine Translation","first-page":"311","author":"Papineni","year":"2002"},{"key":"10.1016\/j.artmed.2026.103421_bb0095","series-title":"Proceedings of the 58th annual meeting of the Association for Computational Linguistics","first-page":"7881","article-title":"BLEURT: learning robust metrics for text generation","author":"Sellam","year":"2020"},{"key":"10.1016\/j.artmed.2026.103421_bb0100","series-title":"Advances in neural information processing systems","article-title":"BARTScore: Evaluating generated text as text generation","author":"Yuan","year":"2021"},{"key":"10.1016\/j.artmed.2026.103421_bb0105","first-page":"65","article-title":"METEOR: an automatic metric for MT evaluation with improved correlation with human judgments","author":"Banerjee","year":"2005","journal-title":"Proceedings of the ACL Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and\/or Summarization"},{"key":"10.1016\/j.artmed.2026.103421_bb0110","series-title":"Proc IEEE Conf Comput Vis Pattern Recognit (CVPR)","first-page":"4566","article-title":"CIDEr: consensus-based image description evaluation","author":"Vedantam","year":"2015"},{"key":"10.1016\/j.artmed.2026.103421_bb0115","series-title":"BERT: Pre-training of deep bidirectional transformers for language understanding. North American chapter of the Association for Computational Linguistics","first-page":"4171","author":"Devlin","year":"2019"},{"key":"10.1016\/j.artmed.2026.103421_bb0120","series-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","first-page":"2575","article-title":"An investigation of evaluation metrics for automated medical note generation","author":"Abacha","year":"2023"},{"key":"10.1016\/j.artmed.2026.103421_bb0125","doi-asserted-by":"crossref","first-page":"163","DOI":"10.1162\/tacl_a_00453","article-title":"SummaC: re-visiting NLI-based models for inconsistency detection in summarization","volume":"10","author":"Laban","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.artmed.2026.103421_bb0130","series-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","first-page":"7580","article-title":"Compression, Transduction, and Creation: A Unified Framework for Evaluating Natural Language Generation","author":"Deng","year":"2021"},{"key":"10.1016\/j.artmed.2026.103421_bb0135","doi-asserted-by":"crossref","first-page":"1995","DOI":"10.1093\/jamia\/ocad177","article-title":"A method to automate the discharge summary hospital course for neurology patients","volume":"30","author":"Hartman","year":"2023","journal-title":"Journal of the American Medical Informatics Association\u202f: JAMIA"},{"key":"10.1016\/j.artmed.2026.103421_bb0140","first-page":"2","article-title":"A Meta-evaluation of faithfulness metrics for long-form hospital-course summarization","volume":"219","author":"Adams","year":"2023","journal-title":"Proceedings of machine learning research"},{"key":"10.1016\/j.artmed.2026.103421_bb0145","doi-asserted-by":"crossref","first-page":"D267","DOI":"10.1093\/nar\/gkh061","article-title":"The unified medical language system (UMLS): integrating biomedical terminology","volume":"32","author":"Bodenreider","year":"2004","journal-title":"Nucleic Acids Res"},{"key":"10.1016\/j.artmed.2026.103421_bb0150","series-title":"The 61st annual meeting of the Association for Computational Linguistics","first-page":"11328","article-title":"AlignScore: Evaluating factual consistency with a unified alignment function","author":"Zha","year":"2023"},{"key":"10.1016\/j.artmed.2026.103421_bb0155","first-page":"10520","article-title":"What are the desired characteristics of calibration sets? Identifying correlates on long form scientific summarization","volume":"2023","author":"Adams","year":"2023","journal-title":"Proc Conf Assoc Comput Linguist Meet"},{"key":"10.1016\/j.artmed.2026.103421_bb0160","article-title":"RadGraph: extracting clinical entities and relations from radiology reports","volume":"101","author":"Jain","year":"2021","journal-title":"PhysioNet"},{"key":"10.1016\/j.artmed.2026.103421_bb0165","series-title":"Findings of the Association for Computational Linguistics: NAACL 2024","first-page":"352","article-title":"Comparing two model designs for clinical note generation; is an LLM a useful evaluator of consistency?","author":"Brake","year":"2024"},{"key":"10.1016\/j.artmed.2026.103421_bb0170","first-page":"897","article-title":"MED-OMIT: extrinsically-focused evaluation metric for omissions in medical summarization","author":"Schumacher","year":"2024","journal-title":"Proceedings of Machine Learning Research"},{"key":"10.1016\/j.artmed.2026.103421_bb0175","series-title":"EMNLP 2024\u20132024 Conference on Empirical Methods in Natural Language Processing, Proceedings of the Conference","first-page":"20061","article-title":"SYNFAC-EDIT: synthetic imitation edit feedback for factual alignment in clinical summarization","author":"Mishra","year":"2024"},{"key":"10.1016\/j.artmed.2026.103421_bb0180","doi-asserted-by":"crossref","DOI":"10.3399\/BJGPO.2023.0116","article-title":"Transforming healthcare documentation: harnessing the potential of AI to generate discharge summaries","volume":"8","author":"Clough","year":"2024","journal-title":"BJGP Open"},{"key":"10.1016\/j.artmed.2026.103421_bb0185","doi-asserted-by":"crossref","first-page":"164","DOI":"10.4338\/ACI-2011-11-RA-0070","article-title":"Assessing electronic note quality using the physician documentation quality instrument (PDQI-9)","volume":"3","author":"Stetson","year":"2012","journal-title":"Appl Clin Inform"},{"key":"10.1016\/j.artmed.2026.103421_bb0190","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2024.112913","article-title":"Context-enhanced framework for medical image report generation using multimodal contexts","volume":"310","author":"Li","year":"2025","journal-title":"Knowl-Based Syst"},{"key":"10.1016\/j.artmed.2026.103421_bb0195","doi-asserted-by":"crossref","first-page":"3786","DOI":"10.1109\/TNNLS.2021.3099165","article-title":"Medical-VLBERT: medical visual language BERT for COVID-19 CT report generation with alternate learning","volume":"32","author":"Liu","year":"2021","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"10.1016\/j.artmed.2026.103421_bb0200","series-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","first-page":"449","article-title":"RadAdapt: radiology report summarization via lightweight domain adaptation of large language models","author":"Van Veen","year":"2023"},{"key":"10.1016\/j.artmed.2026.103421_bb0205","doi-asserted-by":"crossref","first-page":"330","DOI":"10.1177\/00912174241284730","article-title":"Application of artificial intelligence (AI) in the creation of discharge summaries in psychiatric clinics","volume":"60","author":"Janota","year":"2025","journal-title":"Int J Psychiatry Med"},{"key":"10.1016\/j.artmed.2026.103421_bb0210","doi-asserted-by":"crossref","first-page":"1456911","DOI":"10.3389\/fdgth.2024.1456911","article-title":"Comparative study of Claude 3.5-sonnet and human physicians in generating discharge summaries for patients with renal insufficiency: assessment of efficiency, accuracy, and quality","volume":"6","author":"Jin","year":"2024","journal-title":"Front Digital Health"},{"key":"10.1016\/j.artmed.2026.103421_bb0215","doi-asserted-by":"crossref","DOI":"10.1016\/j.ijporl.2025.112275","article-title":"Transforming pediatric ENT documentation: efficiency, accuracy, and adoption of speech recognition technology (Speaknosis)","volume":"191","author":"Langdon","year":"2025","journal-title":"Int J Pediatr Otorhinolaryngol"},{"key":"10.1016\/j.artmed.2026.103421_bb0220","series-title":"EMNLP 2023\u20132023 Conference on Empirical Methods in Natural Language Processing, Proceedings","first-page":"10915","article-title":"FAMESUMM: investigating and improving faithfulness of medical summarization","author":"Zhang","year":"2023"},{"key":"10.1016\/j.artmed.2026.103421_bb0225","article-title":"Data augmented large language models for medical record generation: data augmented large language","volume":"55","author":"Zhang","year":"2025","journal-title":"Appl Intell"},{"key":"10.1016\/j.artmed.2026.103421_bb0230","series-title":"ChatGPT 5 [Large Language Model]","author":"OpenAI","year":"2025"},{"key":"10.1016\/j.artmed.2026.103421_bb0235","series-title":"Cursor [Coding Assistant]","author":"Anysphere","year":"2025"}],"container-title":["Artificial Intelligence in Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0933365726000734?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0933365726000734?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T02:39:53Z","timestamp":1776911993000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0933365726000734"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7]]},"references-count":46,"alternative-id":["S0933365726000734"],"URL":"https:\/\/doi.org\/10.1016\/j.artmed.2026.103421","relation":{},"ISSN":["0933-3657"],"issn-type":[{"value":"0933-3657","type":"print"}],"subject":[],"published":{"date-parts":[[2026,7]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Measuring the quality of AI-generated clinical notes: A systematic review and experimental benchmark of evaluation methods","name":"articletitle","label":"Article Title"},{"value":"Artificial Intelligence in Medicine","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.artmed.2026.103421","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"103421"}}