{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,7]],"date-time":"2026-06-07T12:34:45Z","timestamp":1780835685447,"version":"3.54.1"},"reference-count":20,"publisher":"Oxford University Press (OUP)","issue":"9","license":[{"start":{"date-parts":[[2023,12,18]],"date-time":"2023-12-18T00:00:00Z","timestamp":1702857600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,9,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objective<\/jats:title>\n                  <jats:p>This study evaluates ChatGPT\u2019s symptom-checking accuracy across a broad range of diseases using the Mayo Clinic Symptom Checker patient service as a benchmark.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Methods<\/jats:title>\n                  <jats:p>We prompted ChatGPT with symptoms of 194 distinct diseases. By comparing its predictions with expectations, we calculated a relative comparative score (RCS) to gauge accuracy.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>ChatGPT\u2019s GPT-4 model achieved an average RCS of 78.8%, outperforming the GPT-3.5-turbo by 10.5%. Some specialties scored above 90%.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Discussion<\/jats:title>\n                  <jats:p>The test set, although extensive, was not exhaustive. Future studies should include a more comprehensive disease spectrum.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>ChatGPT exhibits high accuracy in symptom checking for a broad range of diseases, showcasing its potential as a medical training tool in learning health systems to enhance care quality and address health disparities.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocad245","type":"journal-article","created":{"date-parts":[[2023,12,19]],"date-time":"2023-12-19T00:14:00Z","timestamp":1702944840000},"page":"2084-2088","source":"Crossref","is-referenced-by-count":25,"title":["Benchmarking the symptom-checking capabilities of ChatGPT for a broad range of diseases"],"prefix":"10.1093","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4209-8301","authenticated-orcid":false,"given":"Anjun","family":"Chen","sequence":"first","affiliation":[{"name":"Health Sciences, ELHS Institute , Palo Alto, CA 94306, United States"},{"name":"LHS Tech Forum Initiative, Learning Health Community , Palo Alto, CA 94306, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Drake O","family":"Chen","sequence":"additional","affiliation":[{"name":"LHS Tech Forum Initiative, Learning Health Community , Palo Alto, CA 94306, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lu","family":"Tian","sequence":"additional","affiliation":[{"name":"Department of Biomedical Data Science, Stanford University , Stanford, CA 94305, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2023,12,18]]},"reference":[{"issue":"1","key":"2024082207513687600_ocad245-B1","doi-asserted-by":"crossref","first-page":"118","DOI":"10.1038\/s41746-022-00667-w","article-title":"The diagnostic and triage accuracy of digital and online symptom checker tools: a systematic review","volume":"5","author":"Wallace","year":"2022","journal-title":"NPJ Digit Med"},{"issue":"7","key":"2024082207513687600_ocad245-B2","doi-asserted-by":"crossref","first-page":"e0254088","DOI":"10.1371\/journal.pone.0254088","article-title":"Accuracy of online symptom checkers and the potential impact on service utilisation","volume":"16","author":"Ceney","year":"2021","journal-title":"PLoS One"},{"issue":"7956","key":"2024082207513687600_ocad245-B3","doi-asserted-by":"crossref","first-page":"259","DOI":"10.1038\/s41586-023-05881-4","article-title":"Foundation models for generalist medical artificial intelligence","volume":"616","author":"Moor","year":"2023","journal-title":"Nature"},{"issue":"1","key":"2024082207513687600_ocad245-B4","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1001\/jama.2023.8288","article-title":"Accuracy of a generative artificial intelligence model in a complex diagnostic challenge","volume":"330","author":"Kanjee","year":"2023","journal-title":"JAMA"},{"issue":"2","key":"2024082207513687600_ocad245-B5","doi-asserted-by":"crossref","first-page":"e0000198","DOI":"10.1371\/journal.pdig.0000198","article-title":"Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models","volume":"2","author":"Kung","year":"2023","journal-title":"PLoS Digit Health"},{"issue":"7972","key":"2024082207513687600_ocad245-B6","doi-asserted-by":"crossref","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"Singhal","year":"2023","journal-title":"Nature"},{"issue":"6","key":"2024082207513687600_ocad245-B7","doi-asserted-by":"crossref","first-page":"589","DOI":"10.1001\/jamainternmed.2023.1838","article-title":"Comparing physician and artificial intelligence chatbot responses to patient questions posted to a public social media forum","volume":"183","author":"Ayers","year":"2023","journal-title":"JAMA Intern Med"},{"key":"2024082207513687600_ocad245-B8","author":"Mayo Clinic Symptom Checker"},{"issue":"3","key":"2024082207513687600_ocad245-B9","doi-asserted-by":"crossref","first-page":"505","DOI":"10.1038\/s41591-023-02289-5","article-title":"Editorial. Will ChatGPT transform healthcare?","volume":"29","year":"2023","journal-title":"Nat Med"},{"issue":"2","key":"2024082207513687600_ocad245-B10","doi-asserted-by":"crossref","first-page":"e32875","DOI":"10.2196\/32875","article-title":"Operationalizing and implementing pretrained, large artificial intelligence linguistic models in the US health care system: outlook of Generative Pretrained Transformer 3 (GPT-3) as a service model","volume":"10","author":"Sezgin","year":"2022","journal-title":"JMIR Med Inform"},{"key":"2024082207513687600_ocad245-B11","first-page":"133","author":"Committee on the Learning Health Care System in America; Institute of Medicine","year":"2013"},{"key":"2024082207513687600_ocad245-B12","first-page":"47","author":"National Academy of Medicine","year":"2018"},{"issue":"1","key":"2024082207513687600_ocad245-B13","doi-asserted-by":"crossref","first-page":"17917","DOI":"10.1038\/s41598-022-23011-4","article-title":"Simulation of a machine learning enabled learning health system for risk prediction using synthetic patient data","volume":"12","author":"Chen","year":"2022","journal-title":"Sci Rep"},{"issue":"9","key":"2024082207513687600_ocad245-B14","doi-asserted-by":"crossref","first-page":"e232909","DOI":"10.1001\/jamainternmed.2023.2909","article-title":"Chatbot vs medical student performance on free-response clinical reasoning examinations","volume":"183","author":"Strong","year":"2023","journal-title":"JAMA Intern Med"},{"issue":"8","key":"2024082207513687600_ocad245-B15","doi-asserted-by":"crossref","first-page":"e2325000","DOI":"10.1001\/jamanetworkopen.2023.25000","article-title":"Use of GPT-4 to analyze medical records of patients with extensive investigations and delayed diagnosis","volume":"6","author":"Shea","year":"2023","journal-title":"JAMA Netw Open"},{"issue":"13","key":"2024082207513687600_ocad245-B16","doi-asserted-by":"crossref","first-page":"1715","DOI":"10.1001\/jama.283.13.1715","article-title":"Comparison of vignettes, standardized patients, and chart abstraction: a prospective validation study of 3 methods for measuring quality","volume":"283","author":"Peabody","year":"2000","journal-title":"JAMA"},{"issue":"8016","key":"2024082207513687600_ocad245-B17","doi-asserted-by":"crossref","first-page":"h3480","DOI":"10.1136\/bmj.h3480","article-title":"Evaluation of symptom checkers for self diagnosis and triage: audit study","volume":"351","author":"Semigran","year":"2015","journal-title":"BMJ"},{"issue":"4","key":"2024082207513687600_ocad245-B18","doi-asserted-by":"crossref","first-page":"e053566","DOI":"10.1136\/bmjopen-2021-053566","article-title":"What is the suitability of clinical vignettes in benchmarking the performance of online symptom checkers? An audit study","volume":"12","author":"El-Osta","year":"2022","journal-title":"BMJ Open"},{"issue":"9","key":"2024082207513687600_ocad245-B19","doi-asserted-by":"crossref","first-page":"818","DOI":"10.1001\/jama.2023.15481","article-title":"AI in medicine\u2014JAMA\u2019s focus on clinical outcomes, patient-centered care, quality, and equity","volume":"330","author":"Khera","year":"2023","journal-title":"JAMA"},{"issue":"16","key":"2024082207513687600_ocad245-B20","doi-asserted-by":"crossref","first-page":"1347","DOI":"10.1001\/jama.2023.2771","article-title":"Harnessing the promise of artificial intelligence responsibly","volume":"329","author":"Dorr","year":"2023","journal-title":"JAMA"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/2084\/58868110\/ocad245.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/2084\/58868110\/ocad245.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,22]],"date-time":"2024-08-22T11:27:53Z","timestamp":1724326073000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/31\/9\/2084\/7477862"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,18]]},"references-count":20,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2023,12,18]]},"published-print":{"date-parts":[[2024,9,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocad245","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024,9]]},"published":{"date-parts":[[2023,12,18]]}}}