{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,5]],"date-time":"2026-05-05T02:49:49Z","timestamp":1777949389479,"version":"3.51.4"},"reference-count":22,"publisher":"Springer Science and Business Media LLC","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,5,3]]},"DOI":"10.7759\/cureus.108195","type":"journal-article","created":{"date-parts":[[2026,5,3]],"date-time":"2026-05-03T23:39:42Z","timestamp":1777851582000},"source":"Crossref","is-referenced-by-count":0,"title":["A Clinical Quality-Assurance Panel for Patient-Facing Black-Box Large Language Models in Healthcare: Anchored Forced-Choice Drift Monitoring for Sensitive Communication"],"prefix":"10.7759","author":[{"given":"Yuusuke","family":"Harada","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1001\/jamainternmed.2023.1838","article-title":"Comparing physician and artificial intelligence chatbot responses to patient questions posted to a public social media forum","volume":"183","author":"Ayers JW","year":"2023","unstructured":"Ayers JW, Poliak A, Dredze M, et al.. Comparing physician and artificial intelligence chatbot responses to patient questions posted to a public social media forum. JAMA Intern Med. 2023, 183:589-96. 10.1001\/jamainternmed.2023.1838","journal-title":"JAMA Intern Med"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pdig.0000198","article-title":"Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models","volume":"2","author":"Kung TH","year":"2023","unstructured":"Kung TH, Cheatham M, Medenilla A, et al.. Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models. PLOS Digit Health. 2023, 2:e0000198. 10.1371\/journal.pdig.0000198","journal-title":"PLOS Digit Health"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.2196\/59479","article-title":"The opportunities and risks of large language models in mental health","volume":"11","author":"Lawrence HR","year":"2024","unstructured":"Lawrence HR, Schneider RA, Rubin SB, Matari\u0107 MJ, McDuff DJ, Jones Bell M. The opportunities and risks of large language models in mental health. JMIR Ment Health. 2024, 11:e59479. 10.2196\/59479","journal-title":"JMIR Ment Health"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.2196\/mental.7785","article-title":"Delivering cognitive behavior therapy to young adults with symptoms of depression and anxiety using a fully automated conversational agent (Woebot): a randomized controlled trial","volume":"4","author":"Fitzpatrick KK","year":"2017","unstructured":"Fitzpatrick KK, Darcy A, Vierhile M. Delivering cognitive behavior therapy to young adults with symptoms of depression and anxiety using a fully automated conversational agent (Woebot): a randomized controlled trial. JMIR Ment Health. 2017, 4:e19. 10.2196\/mental.7785","journal-title":"JMIR Ment Health"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-023-00873-0","article-title":"The imperative for regulatory oversight of large language models (or generative AI) in healthcare","volume":"6","author":"Mesk\u00f3 B","year":"2023","unstructured":"Mesk\u00f3 B, Topol EJ. The imperative for regulatory oversight of large language models (or generative AI) in healthcare. NPJ Digit Med. 2023, 6:120. 10.1038\/s41746-023-00873-0","journal-title":"NPJ Digit Med"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMp1714229","article-title":"Implementing machine learning in health care - addressing ethical challenges","volume":"378","author":"Char DS","year":"2018","unstructured":"Char DS, Shah NH, Magnus D. Implementing machine learning in health care - addressing ethical challenges. N Engl J Med. 2018, 378:981-3. 10.1056\/NEJMp1714229","journal-title":"N Engl J Med"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-3020","article-title":"\u201cWhy should I trust you?\u201d: explaining the predictions of any classifier","author":"Ribeiro MT","year":"2016","unstructured":"Ribeiro MT, Singh S, Guestrin C. \u201cWhy should I trust you?\u201d: explaining the predictions of any classifier. Proceedings of the 2016 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Demonstrations. DeNero J, Finlayson M, Reddy S (ed): Association for Computational Linguistics, San Diego; 2016. 97-101. 10.18653\/v1\/N16-3020"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1805.10820","article-title":"Local rule-based explanations of black box decision systems","author":"Guidotti R","year":"2018","unstructured":"Guidotti R, Monreale A, Ruggieri S, Pedreschi D, Turini F, Giannotti F. Local rule-based explanations of black box decision systems. arXiv. 2018, 1805-10820. 10.48550\/arXiv.1805.10820","journal-title":"arXiv"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442","article-title":"Beyond accuracy: behavioral testing of NLP models with CheckList","author":"Ribeiro MT","year":"2020","unstructured":"Ribeiro MT, Wu T, Guestrin C, Singh S. Beyond accuracy: behavioral testing of NLP models with CheckList. Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. Jurafsky D, Chai, J, Schluter N, Tetreault J (ed): Association for Computational Linguistics, Online; 2020. 4902-12. 10.18653\/v1\/2020.acl-main.442"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.117","article-title":"Evaluating models\u2019 local decision boundaries via contrast sets","author":"Gardner M","year":"2020","unstructured":"Gardner M, Artzi Y, Basmov V, et al.. Evaluating models\u2019 local decision boundaries via contrast sets. Findings of the Association for Computational Linguistics: EMNLP 2020. Cohn T, He Y, Liu Y (ed): Association for Computational Linguistics, Online; 2020. 1307-23. 10.18653\/v1\/2020.findings-emnlp.117"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/39.3-4.324","article-title":"Rank analysis of incomplete block designs: the method of paired comparisons","volume":"39","author":"Bradley RA","year":"1952","unstructured":"Bradley RA, Terry ME. Rank analysis of incomplete block designs: the method of paired comparisons. Biometrika. 1952, 39:324-45. 10.1093\/biomet\/39.3-4.324","journal-title":"Biometrika"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1060","article-title":"Neural metaphor detection in context","author":"Gao C","year":"2018","unstructured":"Gao C, Choi E, Choi Y, Zettlemoyer L. Neural metaphor detection in context. Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing. Riloff E, Chiang D, Hockenmaier J, Tsujii J (ed): Association for Computational Linguistics, Brussels; 2018. 607-613. 10.18653\/v1\/D18-1060"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.167","article-title":"Do prompt-based models really understand the meaning of their prompts?","author":"Webson A","year":"2022","unstructured":"Webson A, Pavlick E.. Do prompt-based models really understand the meaning of their prompts?. Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Carpuat M, de Marneffe MC, Meza Ruiz, Vladimir I (ed): Association for Computational Linguistics, Seattle; 2022. 2300-44. 10.18653\/v1\/2022.naacl-main.167"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.759","article-title":"Rethinking the role of demonstrations: what makes in-context learning work?","author":"Min S","year":"2022","unstructured":"Min S, Lyu X, Holtzman A, Artetxe M, Lewis M, Hajishirzi H, Zettlemoyer L. Rethinking the role of demonstrations: what makes in-context learning work?. Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing. Goldberg Y, Kozareva Z, Zhang Y (ed): Association for Computational Linguistics, Abu Dhabi; 2022. 11048-11064. 10.18653\/v1\/2022.emnlp-main.759"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176344552","article-title":"Bootstrap methods: another look at the Jackknife","volume":"7","author":"Efron B","year":"1979","unstructured":"Efron B. Bootstrap methods: another look at the Jackknife. Ann Statist. 1979, 7:1-26. 10.1214\/aos\/1176344552","journal-title":"Ann Statist"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.3758\/bf03194544","article-title":"The psychometric function: I. Fitting, sampling, and goodness of fit","volume":"63","author":"Wichmann FA","year":"2001","unstructured":"Wichmann FA, Hill NJ. The psychometric function: I. Fitting, sampling, and goodness of fit. Percept Psychophys. 2001, 63:1293-313. 10.3758\/bf03194544","journal-title":"Percept Psychophys"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/BF02294825","article-title":"Measurement invariance, factor analysis and factorial invariance","volume":"58","author":"Meredith W","year":"1993","unstructured":"Meredith W. Measurement invariance, factor analysis and factorial invariance. Psychometrika. 1993, 58:525-43. 10.1007\/BF02294825","journal-title":"Psychometrika"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1177\/109442810031002","article-title":"A review and synthesis of the measurement invariance literature","volume":"3","author":"Vandenberg RJ","year":"2000","unstructured":"Vandenberg RJ, Lance CE. A review and synthesis of the measurement invariance literature. Organ Res Methods. 2000, 3:4-70. 10.1177\/109442810031002","journal-title":"Organ Res Methods"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1207\/S15328007SEM0902_5","article-title":"Evaluating goodness-of-fit indexes for testing measurement invariance","volume":"9","author":"Cheung GW","year":"2002","unstructured":"Cheung GW, Rensvold RB. Evaluating goodness-of-fit indexes for testing measurement invariance. Struct Equ Modeling. 2002, 9:233-55. 10.1207\/S15328007SEM0902_5","journal-title":"Struct Equ Modeling"},{"key":"ref20","article-title":"On calibration of modern neural networks","author":"Guo C","year":"2017","unstructured":"Guo C, Pleiss G, Sun Y, Weinberger KQ. On calibration of modern neural networks. Proc Mach Learn Res. 2017, 1321-1330.","journal-title":"Proc Mach Learn Res"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596","article-title":"Model cards for model reporting","author":"Mitchell M","year":"2019","unstructured":"Mitchell M, Wu S, Zaldivar A, et al.. Model cards for model reporting. Proceedings of the Conference on Fairness, Accountability, and Transparency (FAT* '19). Association for Computing Machinery, New York; 2019. 220-9. 10.1145\/3287560.3287596"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3458723","article-title":"Datasheets for datasets","volume":"64","author":"Gebru T","year":"2021","unstructured":"Gebru T, Morgenstern J, Vecchione B, et al.. Datasheets for datasets. Commun ACM. 2021, 64:86-92. 10.1145\/3458723","journal-title":"Commun ACM"}],"container-title":["Cureus"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.cureus.com\/articles\/485101-a-clinical-quality-assurance-panel-for-patient-facing-black-box-large-language-models-in-healthcare-anchored-forced-choice-drift-monitoring-for-sensitive-communication","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,3]],"date-time":"2026-05-03T23:39:43Z","timestamp":1777851583000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.cureus.com\/articles\/485101-a-clinical-quality-assurance-panel-for-patient-facing-black-box-large-language-models-in-healthcare-anchored-forced-choice-drift-monitoring-for-sensitive-communication"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,3]]},"references-count":22,"URL":"https:\/\/doi.org\/10.7759\/cureus.108195","relation":{},"ISSN":["2168-8184"],"issn-type":[{"value":"2168-8184","type":"print"}],"subject":[],"published":{"date-parts":[[2026,5,3]]}}}