{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,5]],"date-time":"2026-03-05T04:29:41Z","timestamp":1772684981371,"version":"3.50.1"},"reference-count":27,"publisher":"Oxford University Press (OUP)","issue":"6","license":[{"start":{"date-parts":[[2024,4,17]],"date-time":"2024-04-17T00:00:00Z","timestamp":1713312000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/501100003725","name":"National Research Foundation of Korea","doi-asserted-by":"publisher","award":["2022R1A2C201136011"],"award-info":[{"award-number":["2022R1A2C201136011"]}],"id":[{"id":"10.13039\/501100003725","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Center for Human-Caring Nurse Leaders for the Future"},{"DOI":"10.13039\/100009122","name":"Ministry of Education","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009122","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003725","name":"National Research Foundation of Korea","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003725","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,5,20]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objective<\/jats:title>\n                  <jats:p>This study aims to facilitate the creation of quality standardized nursing statements in South Korea\u2019s hospitals using algorithmic generation based on the International Classifications of Nursing Practice (ICNP) and evaluation through Large Language Models.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>We algorithmically generated 15\u00a0972 statements related to acute respiratory care using 117 concepts and concept composition models of ICNP. Human reviewers, Generative Pre-trained Transformers 4.0 (GPT-4.0), and Bio_Clinical Bidirectional Encoder Representations from Transformers (BERT) evaluated the generated statements for validity. The evaluation by GPT-4.0 and Bio_ClinicalBERT was conducted with and without contextual information and training.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>Of the generated statements, 2207 were deemed valid by expert reviewers. GPT-4.0 showed a zero-shot \u00a0AUC of 0.857, which aggravated with contextual information. Bio_ClinicalBERT, after training, significantly improved, reaching an AUC of 0.998.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>Bio_ClinicalBERT effectively validates auto-generated nursing statements, offering a promising solution to enhance and streamline healthcare documentation processes.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocae070","type":"journal-article","created":{"date-parts":[[2024,4,17]],"date-time":"2024-04-17T19:53:09Z","timestamp":1713383589000},"page":"1397-1403","source":"Crossref","is-referenced-by-count":9,"title":["Evaluating the validity of the nursing statements algorithmically generated based on the International Classifications of Nursing Practice for respiratory nursing care using large language models"],"prefix":"10.1093","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5931-7286","authenticated-orcid":false,"given":"Hyeoneui","family":"Kim","sequence":"first","affiliation":[{"name":"College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"The Research Institute of Nursing Science, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"Center for Human-Caring Nurse Leaders for the Future by Brain Korea 21 (BK 21) Four Project, College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"}]},{"given":"Hyewon","family":"Park","sequence":"additional","affiliation":[{"name":"College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"Samsung Medical Center , Seoul, 06351, Republic of Korea"}]},{"given":"Sunghoon","family":"Kang","sequence":"additional","affiliation":[{"name":"The Department of Science Studies, Seoul National University , Seoul, 08826, Republic of Korea"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9993-5062","authenticated-orcid":false,"given":"Jinsol","family":"Kim","sequence":"additional","affiliation":[{"name":"College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"Center for Human-Caring Nurse Leaders for the Future by Brain Korea 21 (BK 21) Four Project, College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"}]},{"given":"Jeongha","family":"Kim","sequence":"additional","affiliation":[{"name":"College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"Asan Medical Center , Seoul, 05505, Republic of Korea"}]},{"given":"Jinsun","family":"Jung","sequence":"additional","affiliation":[{"name":"College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"},{"name":"Center for Human-Caring Nurse Leaders for the Future by Brain Korea 21 (BK 21) Four Project, College of Nursing, Seoul National University , Seoul, 03080, Republic of Korea"}]},{"given":"Ricky","family":"Taira","sequence":"additional","affiliation":[{"name":"The Department of Radiological Science, David Geffen School of Medicine, University of California Los Angeles , Los Angeles, CA 90095, United States"}]}],"member":"286","published-online":{"date-parts":[[2024,4,17]]},"reference":[{"key":"2024052019463190500_ocae070-B1","author":"International SNOMED","year":"2023"},{"key":"2024052019463190500_ocae070-B2","first-page":"446","article-title":"Building SNOMED CT post-coordinated expressions from annotation groups","volume":"235","author":"Mi\u00f1arro-Gim\u00e9nez","year":"2017","journal-title":"Stud Health Technol Inform"},{"key":"2024052019463190500_ocae070-B3","first-page":"1048","article-title":"Does SNOMED CT post-coordination scale?","volume":"205","author":"Karlsson","year":"2014","journal-title":"Stud Health Technol Inform"},{"issue":"5","key":"2024052019463190500_ocae070-B4","doi-asserted-by":"crossref","first-page":"536","DOI":"10.1197\/jamia.M2093","article-title":"Use of SNOMED CT to represent clinical research data: a semantic characterization of data items on case report forms in vasculitis research","volume":"13","author":"Richesson","year":"2006","journal-title":"J Am Med Inform Assoc"},{"issue":"11","key":"2024052019463190500_ocae070-B5","doi-asserted-by":"crossref","first-page":"1878","DOI":"10.1093\/jamia\/ocad156","article-title":"Future advancement of health care through standardized nursing terminologies: reflections from a Friends of the National Library of Medicine workshop honoring Virginia K. Saba","volume":"30","author":"Dunn Lopez","year":"2023","journal-title":"J Am Med Inform Assoc"},{"issue":"6","key":"2024052019463190500_ocae070-B6","doi-asserted-by":"crossref","first-page":"1126","DOI":"10.1111\/jnu.12894","article-title":"Impact of standardized nursing terminologies on patient and organizational outcomes: a systematic review and meta-analysis","volume":"55","author":"Bertocchi","year":"2023","journal-title":"J Nurs Scholarsh"},{"key":"2024052019463190500_ocae070-B7","doi-asserted-by":"crossref","first-page":"104431","DOI":"10.1016\/j.ijmedinf.2021.104431","article-title":"Use of standardized terminologies in clinical practice: a scoping review","volume":"149","author":"Fennelly","year":"2021","journal-title":"Int J Med Inform"},{"issue":"12","key":"2024052019463190500_ocae070-B8","doi-asserted-by":"crossref","first-page":"1895","DOI":"10.1093\/jamia\/ocad161","article-title":"The suitability of UMLS and SNOMED-CT for encoding outcome concepts","volume":"30","author":"Newbury","year":"2023","journal-title":"J Am Med Inform Assoc"},{"key":"2024052019463190500_ocae070-B9","doi-asserted-by":"crossref","first-page":"104954","DOI":"10.1016\/j.ijmedinf.2022.104954","article-title":"Nursing process, derived from the clinical care classification system components, as an earlier indicator of nursing care during a pandemic","volume":"173","author":"Fine","year":"2023","journal-title":"Int J Med Inform"},{"key":"2024052019463190500_ocae070-B10","doi-asserted-by":"crossref","first-page":"103523","DOI":"10.1016\/j.ijnurstu.2020.103523","article-title":"Use of electronic health records and standardized terminologies: A nationwide survey of nursing staff experiences","volume":"104","author":"De Groot","year":"2020","journal-title":"Int J Nurs Stud"},{"issue":"11","key":"2024052019463190500_ocae070-B11","doi-asserted-by":"crossref","first-page":"1826","DOI":"10.1093\/jamia\/ocad145","article-title":"Utilizing standardized nursing terminologies in implementing an AI-powered fall-prevention tool to improve patient outcomes: a multihospital study","volume":"30","author":"Cho","year":"2023","journal-title":"J Am Med Inform Assoc"},{"key":"2024052019463190500_ocae070-B12","doi-asserted-by":"crossref","first-page":"104968","DOI":"10.1016\/j.ijmedinf.2022.104968","article-title":"Use of the clinical care classification in South Korean nursing practice: challenges and opportunities","volume":"170","author":"Park","year":"2023","journal-title":"Int J Med Inform"},{"key":"2024052019463190500_ocae070-B13","author":"Nurses I.C.O","year":"2023"},{"issue":"2","key":"2024052019463190500_ocae070-B14","doi-asserted-by":"crossref","first-page":"239","DOI":"10.1111\/inr.12566","article-title":"Development of ICNP-based inpatient falls prevention catalogue","volume":"67","author":"Cho","year":"2020","journal-title":"Int Nurs Rev"},{"issue":"4-5","key":"2024052019463190500_ocae070-B15","doi-asserted-by":"crossref","first-page":"304","DOI":"10.1016\/j.jbi.2003.09.004","article-title":"Development and evaluation of a terminology-based electronic nursing record system","volume":"36","author":"Cho","year":"2003","journal-title":"J Biomed Inform"},{"issue":"10","key":"2024052019463190500_ocae070-B16","doi-asserted-by":"crossref","first-page":"1622","DOI":"10.1093\/jamia\/ocad129","article-title":"Predicting emergency department visits and hospitalizations for patients with heart failure in home healthcare using a time series risk model","volume":"30","author":"Chae","year":"2023","journal-title":"J Am Med Inform Assoc"},{"issue":"2","key":"2024052019463190500_ocae070-B17","first-page":"456","article-title":"Prediction of multiclass surgical outcomes in glaucoma using multimodal deep learning based on free-text operative notes and structured EHR data","volume":"31","author":"Lin","year":"2024"},{"key":"2024052019463190500_ocae070-B18","doi-asserted-by":"crossref","first-page":"104442","DOI":"10.1016\/j.jbi.2023.104442","article-title":"AD-BERT: using pre-trained language model to predict the progression from mild cognitive impairment to Alzheimer\u2019s disease","volume":"144","author":"Mao","year":"2023","journal-title":"J Biomed Inform"},{"issue":"1","key":"2024052019463190500_ocae070-B19","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1007\/s11255-023-03773-0","article-title":"Evaluating the performance of ChatGPT in answering questions related to urolithiasis","volume":"56","author":"Cakir","year":"2023","journal-title":"Int Urol Nephrol"},{"issue":"7954","key":"2024052019463190500_ocae070-B20","doi-asserted-by":"crossref","first-page":"773","DOI":"10.1038\/d41586-023-00816-5","article-title":"GPT-4 is here: what scientists think","volume":"615","author":"Sanderson","year":"2023","journal-title":"Nature."},{"issue":"8","key":"2024052019463190500_ocae070-B21","doi-asserted-by":"crossref","first-page":"e2325000","DOI":"10.1001\/jamanetworkopen.2023.25000","article-title":"Use of GPT-4 to analyze medical records of patients with extensive investigations and delayed diagnosis","volume":"6","author":"Shea","year":"2023","journal-title":"JAMA Netw Open"},{"issue":"4","key":"2024052019463190500_ocae070-B22","doi-asserted-by":"crossref","first-page":"2081","DOI":"10.1007\/s00405-023-08104-8","article-title":"Exploring the potential of Chat-GPT as a supportive tool for sialendoscopy clinical decision making and patient information support","volume":"281","author":"Chiesa-Estomba","year":"2024","journal-title":"Eur Arch Otorhinolaryngol"},{"issue":"7","key":"2024052019463190500_ocae070-B23","doi-asserted-by":"crossref","first-page":"1237","DOI":"10.1093\/jamia\/ocad072","article-title":"Using AI-generated suggestions from ChatGPT to optimize clinical decision support","volume":"30","author":"Liu","year":"2023","journal-title":"J Am Med Inform Assoc"},{"key":"2024052019463190500_ocae070-B24","doi-asserted-by":"crossref","first-page":"105173","DOI":"10.1016\/j.ijmedinf.2023.105173","article-title":"Performance and exploration of ChatGPT in medical examination, records and education in Chinese: pave the way for medical AI","volume":"177","author":"Wang","year":"2023","journal-title":"Int J Med Inform"},{"issue":"10","key":"2024052019463190500_ocae070-B25","doi-asserted-by":"crossref","first-page":"1459","DOI":"10.1001\/jamaoncol.2023.2954","article-title":"Use of artificial intelligence chatbots for cancer treatment information","volume":"9","author":"Chen","year":"2023","journal-title":"JAMA Oncol"},{"issue":"3","key":"2024052019463190500_ocae070-B26","doi-asserted-by":"crossref","first-page":"276","DOI":"10.11613\/BM.2012.031","article-title":"Interrater reliability: the kappa statistic","volume":"22","author":"McHugh","year":"2012","journal-title":"Biochem Med"},{"key":"2024052019463190500_ocae070-B27","author":"Shrivastava","year":"2023"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/6\/1397\/57768996\/ocae070.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/6\/1397\/57768996\/ocae070.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,20]],"date-time":"2024-05-20T19:47:22Z","timestamp":1716234442000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/31\/6\/1397\/7648662"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,17]]},"references-count":27,"journal-issue":{"issue":"6","published-online":{"date-parts":[[2024,4,17]]},"published-print":{"date-parts":[[2024,5,20]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocae070","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024,6,1]]},"published":{"date-parts":[[2024,4,17]]}}}