{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,7]],"date-time":"2026-10-07T11:41:22Z","timestamp":1791373282017,"version":"4.3.1"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Indian Institute of Technology Kharagpur","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Federal Ministry of Education and Research (BMBF), Germany","award":["01DD20003"],"award-info":[{"award-number":["01DD20003"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657882","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"1073-1082","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":18,"title":["Beyond Accuracy: Investigating Error Types in GPT-4 Responses to USMLE Questions"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7269-2163","authenticated-orcid":false,"given":"Soumyadeep","family":"Roy","sequence":"first","affiliation":[{"name":"IIT Kharagpur, Kharagpur, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8235-1637","authenticated-orcid":false,"given":"Aparup","family":"Khatua","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-1814-8513","authenticated-orcid":false,"given":"Fatemeh","family":"Ghoochani","sequence":"additional","affiliation":[{"name":"L3S Research Center, Hannover, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8122-7572","authenticated-orcid":false,"given":"Uwe","family":"Hadler","sequence":"additional","affiliation":[{"name":"L3S Research Center, Hannover, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3374-2193","authenticated-orcid":false,"given":"Wolfgang","family":"Nejdl","sequence":"additional","affiliation":[{"name":"L3S Research Center, Hannover, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3967-186X","authenticated-orcid":false,"given":"Niloy","family":"Ganguly","sequence":"additional","affiliation":[{"name":"IIT Kharagpur, Kharagpur, India"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Machine Learning for Healthcare Conference. PMLR, 2--30","author":"Adams Griffin","year":"2023","unstructured":"Griffin Adams, Jason Zuckerg, and No\u00e9mie Elhadad. 2023. A meta-evaluation of faithfulness metrics for long-form hospital-course summarization. In Machine Learning for Healthcare Conference. PMLR, 2--30."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i16.29720"},{"key":"e_1_3_2_1_3_1","volume-title":"Benchmarking the symptomchecking capabilities of ChatGPT for a broad range of diseases. Journal of the American Medical Informatics Association (12","author":"Chen Anjun","year":"2023","unstructured":"Anjun Chen, Drake O Chen, and Lu Tian. 2023. Benchmarking the symptomchecking capabilities of ChatGPT for a broad range of diseases. Journal of the American Medical Informatics Association (12 2023), ocad245. https:\/\/doi.org\/10. 1093\/jamia\/ocad245"},{"key":"e_1_3_2_1_4_1","volume-title":"How is ChatGPT's behavior changing over time? arXiv preprint arXiv:2307.09009","author":"Chen Lingjiao","year":"2023","unstructured":"Lingjiao Chen, Matei Zaharia, and James Zou. 2023. How is ChatGPT's behavior changing over time? arXiv preprint arXiv:2307.09009 (2023)."},{"key":"e_1_3_2_1_5_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun et al. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374 [cs.LG]"},{"key":"e_1_3_2_1_6_1","unstructured":"I Chern Steffi Chern Shiqi Chen et al. 2023. FacTool: Factuality Detection in Generative AI--A Tool Augmented Framework for Multi-Task and Multi-Domain Scenarios. arXiv preprint arXiv:2307.13528 (2023)."},{"key":"e_1_3_2_1_7_1","unstructured":"Debadutta Dash Rahul Thapa Juan M Banda et al. 2023. Evaluation of GPT-3.5 and GPT-4 for supporting real-world information needs in healthcare delivery. arXiv preprint arXiv:2304.13714 (2023)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.415"},{"key":"e_1_3_2_1_9_1","unstructured":"Tianyu Han Lisa C. Adams Jens-Michalis Papaioannou et al. 2023. MedAlpaca - An Open-Source Collection of Medical Conversational AI Models and Training Data. arXiv:2304.08247 [cs.CL]"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-short.28"},{"key":"e_1_3_2_1_11_1","volume-title":"Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=d7KBjmI3GmQ","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=d7KBjmI3GmQ"},{"key":"e_1_3_2_1_12_1","unstructured":"Lei Huang Weijiang Yu Weitao Ma et al. 2023. A survey on hallucination in large language models: Principles taxonomy challenges and open questions. arXiv preprint arXiv:2311.05232 (2023)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11146421"},{"key":"e_1_3_2_1_14_1","unstructured":"Qiao Jin Fangyuan Chen Yiliang Zhou et al. 2024. Hidden flaws behind expertlevel accuracy of gpt-4 vision in medicine. arXiv preprint arXiv:2401.08396 (2024)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1259"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Alistair EW Johnson Tom J Pollard Seth J Berkowitz et al. 2019. MIMIC-CXR a de-identified publicly available database of chest radiographs with free-text reports. Scientific data 6 1 (2019) 317.","DOI":"10.1038\/s41597-019-0322-0"},{"key":"e_1_3_2_1_17_1","volume-title":"Who answers it better? an in-depth analysis of ChatGPT and stack overflow answers to software engineering questions. arXiv preprint arXiv:2308.02312","author":"Kabir Samia","year":"2023","unstructured":"Samia Kabir, David N Udo-Imeh, Bonan Kou, and Tianyi Zhang. 2023. Who answers it better? an in-depth analysis of ChatGPT and stack overflow answers to software engineering questions. arXiv preprint arXiv:2308.02312 (2023)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.2023.8288"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12859-020-3517-7"},{"key":"e_1_3_2_1_20_1","volume-title":"Construction of Depression Knowledge Graph Based on Biomedical Literature. In 2021 IEEE International Conference on Bioinformatics and Biomedicine (BIBM). IEEE","author":"Li Zepeng","year":"2021","unstructured":"Zepeng Li, Yufeng Zhang, Rikui Huang, et al. 2021. Construction of Depression Knowledge Graph Based on Biomedical Literature. In 2021 IEEE International Conference on Bioinformatics and Biomedicine (BIBM). IEEE, 1849--1855."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2024.100943"},{"key":"e_1_3_2_1_22_1","unstructured":"Daniel McDuff Mike Schaekermann Tao Tu et al. 2023. Towards Accurate Differential Diagnosis with Large Language Models. arXiv:2312.00164 [cs.CY]"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.394"},{"key":"e_1_3_2_1_24_1","volume-title":"Dean Carignan, and Eric Horvitz.","author":"Nori Harsha","year":"2023","unstructured":"Harsha Nori, Nicholas King, Scott Mayer McKinney, Dean Carignan, and Eric Horvitz. 2023. Capabilities of gpt-4 on medical challenge problems. arXiv preprint arXiv:2303.13375 (2023)."},{"key":"e_1_3_2_1_25_1","unstructured":"OpenAI : Josh Achiam Steven Adler et al. 2023. GPT-4 Technical Report. arXiv:2303.08774 [cs.CL]"},{"key":"e_1_3_2_1_26_1","volume-title":"Proceedings of the Conference on Health, Inference, and Learning (Proceedings of Machine Learning Research","volume":"260","author":"Pal Ankit","year":"2022","unstructured":"Ankit Pal, Logesh Kumar Umapathi, and Malaikannan Sankarasubbu. 2022. MedMCQA: A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question Answering. In Proceedings of the Conference on Health, Inference, and Learning (Proceedings of Machine Learning Research, Vol. 174). 248--260. https:\/\/proceedings.mlr.press\/v174\/pal22a.html"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-demos.33"},{"key":"e_1_3_2_1_28_1","volume-title":"When Do Annotator Demographics Matter? Measuring the Influence of Annotator Demographics with the POPQUORN Dataset. arXiv preprint arXiv:2306.06826","author":"Pei Jiaxin","year":"2023","unstructured":"Jiaxin Pei and David Jurgens. 2023. When Do Annotator Demographics Matter? Measuring the Influence of Annotator Demographics with the POPQUORN Dataset. arXiv preprint arXiv:2306.06826 (2023)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","unstructured":"Cheng Peng Xi Yang Aokun Chen et al. 2023. A study of generative large language model for medical research and healthcare. npj Digital Medicine 6 1 (16 Nov 2023) 210. https:\/\/doi.org\/10.1038\/s41746-023-00958-w","DOI":"10.1038\/s41746-023-00958-w"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531745"},{"key":"e_1_3_2_1_31_1","unstructured":"Prolific 2023. https:\/\/app.prolific.com\/"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2003.11.003"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","unstructured":"Thomas Savage Ashwin Nayak Robert Gallo et al. 2024. Diagnostic reasoning prompts reveal the potential for large language model interpretability inmedicine. npj Digital Medicine 7 1 (24 Jan 2024) 20. https:\/\/doi.org\/10.1038\/s41746-024-01010-1","DOI":"10.1038\/s41746-024-01010-1"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Dalton Schutte Jake Vasilakes Anu Bompelli et al. 2021. Discovering novel drug-supplement interactions using a dietary supplements knowledge graph generated from the biomedical literature. arXiv preprint arXiv:2106.12741 (2021).","DOI":"10.1016\/j.jbi.2022.104120"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"Karan Singhal Shekoofeh Azizi Tao Tu et al. 2023. Large language models encode clinical knowledge. Nature 620 7972 (01 Aug 2023) 172--180. https: \/\/doi.org\/10.1038\/s41586-023-06291-2","DOI":"10.1038\/s41586-023-06291-2"},{"key":"e_1_3_2_1_36_1","unstructured":"Karan Singhal Tao Tu Juraj Gottweis et al. 2023. Towards Expert-Level Medical Question Answering with Large Language Models. arXiv:2305.09617 [cs.CL]"},{"key":"e_1_3_2_1_37_1","volume-title":"Galactica: A Large Language Model for Science. arXiv:2211.09085 [cs.CL]","author":"Taylor Ross","year":"2022","unstructured":"Ross Taylor, Marcin Kardas, Guillem Cucurull, et al. 2022. Galactica: A Large Language Model for Science. arXiv:2211.09085 [cs.CL]"},{"key":"e_1_3_2_1_38_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone et al. 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_39_1","volume-title":"https:\/\/www.usmle.org\/about-usmle Accessed on","author":"USMLE","year":"2024","unstructured":"USMLE official website 2024. https:\/\/www.usmle.org\/about-usmle Accessed on February 7, 2024."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.609"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i12.26698"},{"key":"e_1_3_2_1_42_1","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Schuurmans Dale","year":"2022","unstructured":"JasonWei, XuezhiWang, Dale Schuurmans, et al. 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems 35 (2022), 24824--24837.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_43_1","volume-title":"Flaws in clinical reasoning: a common cause of diagnostic error. American family physician 84, 9","author":"Wellbery Caroline","year":"2011","unstructured":"Caroline Wellbery. 2011. Flaws in clinical reasoning: a common cause of diagnostic error. American family physician 84, 9 (2011), 1042--1048."},{"key":"e_1_3_2_1_44_1","unstructured":"Chaoyi Wu Weixiong Lin Xiaoman Zhang Ya Zhang Yanfeng Wang and Weidi Xie. 2023. PMC-LLaMA: Towards Building Open-source Language Models for Medicine. arXiv:2304.14454 [cs.CL]"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.109460"},{"key":"e_1_3_2_1_46_1","volume-title":"A Survey on Biomedical Text Summarization with Pre-trained Language Model. arXiv preprint arXiv:2304.08763","author":"Xie Qianqian","year":"2023","unstructured":"Qianqian Xie, Zheheng Luo, Benyou Wang, and Sophia Ananiadou. 2023. A Survey on Biomedical Text Summarization with Pre-trained Language Model. arXiv preprint arXiv:2304.08763 (2023)."},{"key":"e_1_3_2_1_47_1","unstructured":"Qianqian Xie Edward J Schenck He S Yang et al. 2023. Faithful ai in medicine: A systematic review with large language models and beyond. medRxiv (2023)."},{"key":"e_1_3_2_1_48_1","unstructured":"Honglin Xiong Sheng Wang Yitao Zhu et al. 2023. DoctorGLM: Fine-tuning your Chinese Doctor is not a Herculean Task. arXiv:2304.01097 [cs.CL]"},{"key":"e_1_3_2_1_49_1","unstructured":"Hattie Zhou Azade Nova Hugo Larochelle Aaron Courville Behnam Neyshabur and Hanie Sedghi. 2022. Teaching Algorithmic Reasoning via In-context Learning. arXiv:2211.09066 [cs.LG]"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.342"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657882","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657882","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:33:47Z","timestamp":1755840827000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657882"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":50,"alternative-id":["10.1145\/3626772.3657882","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657882","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}