{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,3]],"date-time":"2026-08-03T16:49:40Z","timestamp":1785775780332,"version":"3.56.0"},"reference-count":21,"publisher":"Springer Science and Business Media LLC","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,8,3]]},"DOI":"10.7759\/cureus.113882","type":"journal-article","created":{"date-parts":[[2026,8,3]],"date-time":"2026-08-03T16:13:17Z","timestamp":1785773597000},"source":"Crossref","is-referenced-by-count":0,"title":["Large Language Models as Examinees and Graders in Simulated General Surgery Oral Board-Style Cases: A Psychometric Pilot Study"],"prefix":"10.7759","author":[{"given":"Kian A","family":"Huang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haris K","family":"Choudhary","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Allan L","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wid","family":"Alhassani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Samvelian","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ronald J","family":"Weigel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marshall  S","family":"Baker","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fred A","family":"Luchette","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paul  C","family":"Kuo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","reference":[{"key":"ref1","unstructured":"American Board of Surgery. Certifying Examination. Accessed: April 2026: https:\/\/www.absurgery.org\/get-certified\/general-surgery\/certifying-exam\/."},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000004028","article-title":"Measuring the effect of examiner variability in a multiple-circuit objective structured clinical examination (OSCE)","volume":"96","author":"Yeates P","year":"2021","unstructured":"Yeates P, Moult A, Cope N, et al.. Measuring the effect of examiner variability in a multiple-circuit objective structured clinical examination (OSCE). Acad Med. 2021, 96:1189-96. 10.1097\/ACM.0000000000004028","journal-title":"Acad Med"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pdig.0000198","article-title":"Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models","volume":"2","author":"Kung TH","year":"2023","unstructured":"Kung TH, Cheatham M, Medenilla A, et al.. Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models. PLOS Digit Health. 2023, 2:e0000198. 10.1371\/journal.pdig.0000198","journal-title":"PLOS Digit Health"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.2196\/64963","article-title":"Comparing diagnostic accuracy of clinical professionals and large language models: systematic review and meta-analysis","volume":"13","author":"Shan G","year":"2025","unstructured":"Shan G, Chen X, Wang C, Liu L, Gu Y, Jiang H, Shi T. Comparing diagnostic accuracy of clinical professionals and large language models: systematic review and meta-analysis. JMIR Med Inform. 2025, 13:e64963. 10.2196\/64963","journal-title":"JMIR Med Inform"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-024-01391-3","article-title":"SurgeryLLM: a retrieval-augmented generation large language model framework for surgical decision support and workflow enhancement","volume":"7","author":"Ong CS","year":"2024","unstructured":"Ong CS, Obey NT, Zheng Y, Cohan A, Schneider EB. SurgeryLLM: a retrieval-augmented generation large language model framework for surgical decision support and workflow enhancement. NPJ Digit Med. 2024, 7:364. 10.1038\/s41746-024-01391-3","journal-title":"NPJ Digit Med"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1097\/00001888-199511000-00026","article-title":"A surgery oral examination: interrater agreement and the influence of rater characteristics","volume":"70","author":"Burchard KW","year":"1995","unstructured":"Burchard KW, Rowland-Morin PA, Coe NP, Garb JL. A surgery oral examination: interrater agreement and the influence of rater characteristics. Acad Med. 1995, 70:1044-6. 10.1097\/00001888-199511000-00026","journal-title":"Acad Med"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.3352\/jeehp.2013.10.2","article-title":"Assessment methods in surgical training in the United Kingdom","volume":"10","author":"Evgeniou E","year":"2013","unstructured":"Evgeniou E, Peter L, Tsironi M, Iyer S. Assessment methods in surgical training in the United Kingdom. J Educ Eval Health Prof. 2013, 10:2. 10.3352\/jeehp.2013.10.2","journal-title":"J Educ Eval Health Prof"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2020.05.015","article-title":"Standardized multi-institutional mock oral examination: a feasible and valuable educational experience for general surgery residents","volume":"77","author":"Lu Y","year":"2020","unstructured":"Lu Y, Miranda R, Quach C, Girgis M, Lewis CE, Tillou A, Chen F. Standardized multi-institutional mock oral examination: a feasible and valuable educational experience for general surgery residents. J Surg Educ. 2020, 77:1568-76. 10.1016\/j.jsurg.2020.05.015","journal-title":"J Surg Educ"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"Singhal K","year":"2023","unstructured":"Singhal K, Azizi S, Tu T, et al.. Large language models encode clinical knowledge. Nature. 2023, 620:172-80. 10.1038\/s41586-023-06291-2","journal-title":"Nature"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.3352\/jeehp.2017.14.34","article-title":"The sights and insights of examiners in objective structured clinical examinations","volume":"14","author":"Chong L","year":"2017","unstructured":"Chong L, Taylor S, Haywood M, Adelstein BA, Shulruf B. The sights and insights of examiners in objective structured clinical examinations. J Educ Eval Health Prof. 2017, 14:34. 10.3352\/jeehp.2017.14.34","journal-title":"J Educ Eval Health Prof"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1016\/S0140-6736(15)60160-X","article-title":"Global Surgery 2030: evidence and solutions for achieving health, welfare, and economic development","volume":"386","author":"Meara JG","year":"2015","unstructured":"Meara JG, Leather AJM, Hagander L, et al.. Global Surgery 2030: evidence and solutions for achieving health, welfare, and economic development. Lancet. 2015, 386:569-624. 10.1016\/S0140-6736(15)60160-X","journal-title":"Lancet"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-3456-0","article-title":"Generalizability Theory","author":"Brennan RL","year":"2001","unstructured":"Brennan RL. Generalizability Theory. Springer-Verlag, New York, NY; 2001. 10.1007\/978-1-4757-3456-0"},{"key":"ref13","article-title":"The General Surgery Oral Board Review","author":"Velez DR","year":"2025","unstructured":"Velez DR. The General Surgery Oral Board Review . The Operative Review of Surgery, Orlando, FL; 2025."},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/s10459-010-9222-y","article-title":"Likert scales, levels of measurement and the \"laws\" of statistics","volume":"15","author":"Norman G","year":"2010","unstructured":"Norman G. Likert scales, levels of measurement and the \"laws\" of statistics. Adv Health Sci Educ Theory Pract. 2010, 15:625-32. 10.1007\/s10459-010-9222-y","journal-title":"Adv Health Sci Educ Theory Pract"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1177\/00131644221130482","article-title":"Equidistant response options on Likert-type instruments: testing the interval scaling assumption using Mplus","volume":"83","author":"Sideridis G","year":"2023","unstructured":"Sideridis G, Tsaousis I, Ghamdi H. Equidistant response options on Likert-type instruments: testing the interval scaling assumption using Mplus. Educ Psychol Meas. 2023, 83:885-906. 10.1177\/00131644221130482","journal-title":"Educ Psychol Meas"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1097\/00001888-199009000-00045","article-title":"The assessment of clinical skills\/competence\/performance","volume":"65","author":"Miller GE","year":"1990","unstructured":"Miller GE. The assessment of clinical skills\/competence\/performance. Acad Med. 1990, 65:S63-7. 10.1097\/00001888-199009000-00045","journal-title":"Acad Med"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-024-03423-7","article-title":"Toward expert-level medical question answering with large language models","volume":"31","author":"Singhal K","year":"2025","unstructured":"Singhal K, Tu T, Gottweis J, et al.. Toward expert-level medical question answering with large language models. Nat Med. 2025, 31:943-50. 10.1038\/s41591-024-03423-7","journal-title":"Nat Med"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/s10459-012-9372-1","article-title":"Seeing the same thing differently: mechanisms that contribute to assessor differences in directly-observed performance assessments","volume":"18","author":"Yeates P","year":"2013","unstructured":"Yeates P, O'Neill P, Mann K, Eva K. Seeing the same thing differently: mechanisms that contribute to assessor differences in directly-observed performance assessments. Adv Health Sci Educ Theory Pract. 2013, 18:325-41. 10.1007\/s10459-012-9372-1","journal-title":"Adv Health Sci Educ Theory Pract"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000004150","article-title":"Use of generalizability theory for exploring reliability of and sources of variance in assessment of technical skills: a systematic review and meta-analysis","volume":"96","author":"Andersen SA","year":"2021","unstructured":"Andersen SA, Nayahangan LJ, Park YS, Konge L. Use of generalizability theory for exploring reliability of and sources of variance in assessment of technical skills: a systematic review and meta-analysis. Acad Med. 2021, 96:1609-19. 10.1097\/ACM.0000000000004150","journal-title":"Acad Med"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2923.2004.01755.x","article-title":"Techniques for measuring clinical competence: objective structured clinical examinations","volume":"38","author":"Newble D","year":"2004","unstructured":"Newble D. Techniques for measuring clinical competence: objective structured clinical examinations. Med Educ. 2004, 38:199-203. 10.1111\/j.1365-2923.2004.01755.x","journal-title":"Med Educ"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbusres.2025.115804","article-title":"Jagged competencies: measuring the reliability of generative AI in academic research","volume":"203","author":"Thomas LDW","year":"2026","unstructured":"Thomas LDW, Romasanta AKG, Pujol Priego L. Jagged competencies: measuring the reliability of generative AI in academic research. J Bus Res. 2026, 203:115804. 10.1016\/j.jbusres.2025.115804","journal-title":"J Bus Res"}],"container-title":["Cureus"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.cureus.com\/articles\/508578-large-language-models-as-examinees-and-graders-in-simulated-general-surgery-oral-board-style-cases-a-psychometric-pilot-study","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,3]],"date-time":"2026-08-03T16:13:18Z","timestamp":1785773598000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.cureus.com\/articles\/508578-large-language-models-as-examinees-and-graders-in-simulated-general-surgery-oral-board-style-cases-a-psychometric-pilot-study"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,3]]},"references-count":21,"URL":"https:\/\/doi.org\/10.7759\/cureus.113882","relation":{},"ISSN":["2168-8184"],"issn-type":[{"value":"2168-8184","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8,3]]}}}