{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,19]],"date-time":"2026-02-19T03:22:15Z","timestamp":1771471335262,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,24]],"date-time":"2024-08-24T00:00:00Z","timestamp":1724457600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,25]]},"DOI":"10.1145\/3637528.3671575","type":"proceedings-article","created":{"date-parts":[[2024,8,25]],"date-time":"2024-08-25T04:54:55Z","timestamp":1724561695000},"page":"5466-5475","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["Towards Automatic Evaluation for LLMs' Clinical Capabilities: Metric, Data, and Algorithm"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8109-5248","authenticated-orcid":false,"given":"Lei","family":"Liu","sequence":"first","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7799-8460","authenticated-orcid":false,"given":"Xiaoyan","family":"Yang","sequence":"additional","affiliation":[{"name":"Ant Group, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4999-4662","authenticated-orcid":false,"given":"Fangzhou","family":"Li","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4075-6147","authenticated-orcid":false,"given":"Chenfei","family":"Chi","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1046-9000","authenticated-orcid":false,"given":"Yue","family":"Shen","sequence":"additional","affiliation":[{"name":"Ant Group, Hangzhou, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9493-0601","authenticated-orcid":false,"given":"Shiwei","family":"Lyu","sequence":"additional","affiliation":[{"name":"Ant Group, Hangzhou, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1160-2635","authenticated-orcid":false,"given":"Ming","family":"Zhang","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0105-732X","authenticated-orcid":false,"given":"Xiaowei","family":"Ma","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-5946-0351","authenticated-orcid":false,"given":"Xiangguo","family":"Lv","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1856-9946","authenticated-orcid":false,"given":"Liya","family":"Ma","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2321-7259","authenticated-orcid":false,"given":"Zhiqiang","family":"Zhang","sequence":"additional","affiliation":[{"name":"Ant Group, Hangzhou, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0124-9433","authenticated-orcid":false,"given":"Wei","family":"Xue","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4379-3002","authenticated-orcid":false,"given":"Yiran","family":"Huang","sequence":"additional","affiliation":[{"name":"Renji Hospital, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7596-4945","authenticated-orcid":false,"given":"Jinjie","family":"Gu","sequence":"additional","affiliation":[{"name":"Ant Group, Hangzhou, China"}]}],"member":"320","published-online":{"date-parts":[[2024,8,24]]},"reference":[{"key":"e_1_3_2_2_2_1","volume-title":"Baichuan 2: Open Large-scale Language Models. arXiv preprint arXiv:2309.10305","year":"2023","unstructured":"Baichuan. 2023. Baichuan 2: Open Large-scale Language Models. arXiv preprint arXiv:2309.10305 (2023). https:\/\/arxiv.org\/abs\/2309.10305"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"Erin P Balogh Bryan T Miller and John R Ball. 2015. Improving diagnosis in health care. (2015).","DOI":"10.17226\/21794"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1097\/00001888-199306000-00002"},{"key":"e_1_3_2_2_5_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language models are few-shot learners. Advances in neural information processing systems Vol. 33 (2020) 1877--1901."},{"key":"e_1_3_2_2_6_1","unstructured":"Junying Chen Xidong Wang Anningzhe Gao Feng Jiang Shunian Chen Hongbo Zhang Dingjie Song Wenya Xie Chuyi Kong Jianquan Li et al. 2023. HuatuoGPT-II One-stage Training for Medical Adaption of LLMs. arXiv preprint arXiv:2311.09774 (2023)."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"crossref","unstructured":"Jeffrey De Fauw Joseph R Ledsam Bernardino Romera-Paredes Stanislav Nikolov Nenad Tomasev Sam Blackwell Harry Askham Xavier Glorot Brendan O'Donoghue Daniel Visentin et al. 2018. Clinically applicable deep learning for diagnosis and referral in retinal disease. Nature medicine Vol. 24 9 (2018) 1342--1350.","DOI":"10.1038\/s41591-018-0107-6"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.2196\/45312"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.2196\/48163"},{"key":"e_1_3_2_2_11_1","volume-title":"What is a clinical pathway? Development of a definition to inform the debate. BMC medicine","author":"Kinsman Leigh","year":"2010","unstructured":"Leigh Kinsman, Thomas Rotter, Erica James, Pamela Snow, and Jon Willis. 2010. What is a clinical pathway? Development of a definition to inform the debate. BMC medicine, Vol. 8, 1 (2010), 1--3."},{"key":"e_1_3_2_2_12_1","volume-title":"Camille Elepa no, Maria Madriaga, Rimel Aggabao, Giezel Diaz-Candido, James Maningo, et al.","author":"Kung Tiffany H","year":"2023","unstructured":"Tiffany H Kung, Morgan Cheatham, Arielle Medenilla, Czarina Sillos, Lorie De Leon, Camille Elepa no, Maria Madriaga, Rimel Aggabao, Giezel Diaz-Candido, James Maningo, et al. 2023. Performance of ChatGPT on USMLE: Potential for AI-assisted medical education using large language models. PLoS digital health, Vol. 2, 2 (2023), e0000198."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-016-0580-z"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMsr2214184"},{"key":"e_1_3_2_2_15_1","volume-title":"Expert: Modeling Medical Knowledge into General LLMs. arXiv preprint arXiv:2312.01040","author":"Li Qiang","year":"2023","unstructured":"Qiang Li, Xiaoyan Yang, Haowen Wang, Qin Wang, Lei Liu, Junjie Wang, Yang Zhang, Mingyuan Chu, Sen Hu, Yicheng Chen, et al. 2023. From Beginner to Expert: Modeling Medical Knowledge into General LLMs. arXiv preprint arXiv:2312.01040 (2023)."},{"key":"e_1_3_2_2_16_1","volume-title":"An Automatic Evaluation Framework for Multi-turn Medical Consultations Capabilities of Large Language Models. arXiv preprint arXiv:2309.02077","author":"Liao Yusheng","year":"2023","unstructured":"Yusheng Liao, Yutong Meng, Hongcheng Liu, Yanfeng Wang, and Yu Wang. 2023. An Automatic Evaluation Framework for Multi-turn Medical Consultations Capabilities of Large Language Models. arXiv preprint arXiv:2309.02077 (2023)."},{"key":"e_1_3_2_2_17_1","volume-title":"Conference on Health, Inference, and Learning. PMLR, 248--260","author":"Pal Ankit","year":"2022","unstructured":"Ankit Pal, Logesh Kumar Umapathi, and Malaikannan Sankarasubbu. 2022. Medmcqa: A large-scale multi-subject multi-choice dataset for medical domain question answering. In Conference on Health, Inference, and Learning. PMLR, 248--260."},{"key":"e_1_3_2_2_18_1","volume-title":"Llm-mini-cex: Automatic evaluation of large language model for diagnostic conversation. arXiv preprint arXiv:2308.07635","author":"Shi Xiaoming","year":"2023","unstructured":"Xiaoming Shi, Jie Xu, Jinru Ding, Jiali Pang, Sichen Liu, Shuqing Luo, Xingwei Peng, Lu Lu, Haihong Yang, Mingtao Hu, et al. 2023. Llm-mini-cex: Automatic evaluation of large language model for diagnostic conversation. arXiv preprint arXiv:2308.07635 (2023)."},{"key":"e_1_3_2_2_19_1","volume-title":"Nathan Scales, Ajay Tanwani, Heather Cole-Lewis, Stephen Pfohl, et al.","author":"Singhal Karan","year":"2023","unstructured":"Karan Singhal, Shekoofeh Azizi, Tao Tu, S Sara Mahdavi, Jason Wei, Hyung Won Chung, Nathan Scales, Ajay Tanwani, Heather Cole-Lewis, Stephen Pfohl, et al. 2023. Large language models encode clinical knowledge. Nature, Vol. 620, 7972 (2023), 172--180."},{"key":"e_1_3_2_2_20_1","unstructured":"Karan Singhal Tao Tu Juraj Gottweis Rory Sayres Ellery Wulczyn Le Hou Kevin Clark Stephen Pfohl Heather Cole-Lewis Darlene Neal et al. 2023. Towards expert-level medical question answering with large language models. arXiv preprint arXiv:2305.09617 (2023)."},{"key":"e_1_3_2_2_21_1","volume-title":"Nature","volume":"614","author":"Stokel-Walker Chris","year":"2023","unstructured":"Chris Stokel-Walker and Richard Van Noorden. 2023. What ChatGPT and generative AI mean for science. Nature, Vol. 614, 7947 (2023), 214--216."},{"key":"e_1_3_2_2_22_1","volume-title":"Chatdoctor: A medical chat model fine-tuned on llama model using medical domain knowledge. arXiv preprint arXiv:2303.14070","author":"Yunxiang Li","year":"2023","unstructured":"Li Yunxiang, Li Zihan, Zhang Kai, Dan Ruilong, and Zhang You. 2023. Chatdoctor: A medical chat model fine-tuned on llama model using medical domain knowledge. arXiv preprint arXiv:2303.14070 (2023)."},{"key":"e_1_3_2_2_23_1","volume-title":"Almanac: Retrieval-augmented language models for clinical medicine. Research Square","author":"Zakka Cyril","year":"2023","unstructured":"Cyril Zakka, Akash Chaurasia, Rohan Shad, Alex R Dalal, Jennifer L Kim, Michael Moor, Kevin Alexander, Euan Ashley, Jack Boyd, Kathleen Boyd, et al. 2023. Almanac: Retrieval-augmented language models for clinical medicine. Research Square (2023)."},{"key":"e_1_3_2_2_24_1","unstructured":"Aohan Zeng Xiao Liu Zhengxiao Du Zihan Wang Hanyu Lai Ming Ding Zhuoyi Yang Yifan Xu Wendi Zheng Xiao Xia et al. 2022. Glm-130b: An open bilingual pre-trained model. arXiv preprint arXiv:2210.02414 (2022)."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"crossref","unstructured":"Hongbo Zhang Junying Chen Feng Jiang Fei Yu Zhihong Chen Jianquan Li Guiming Chen Xiangbo Wu Zhiyi Zhang Qingying Xiao et al. 2023. HuatuoGPT towards Taming Language Model to Be a Doctor. arXiv preprint arXiv:2305.15075 (2023).","DOI":"10.18653\/v1\/2023.findings-emnlp.725"},{"key":"e_1_3_2_2_26_1","volume-title":"How language model hallucinations can snowball. arXiv preprint arXiv:2305.13534","author":"Zhang Muru","year":"2023","unstructured":"Muru Zhang, Ofir Press, William Merrill, Alisa Liu, and Noah A Smith. 2023. How language model hallucinations can snowball. arXiv preprint arXiv:2305.13534 (2023)."}],"event":{"name":"KDD '24: The 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Barcelona Spain","acronym":"KDD '24","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3637528.3671575","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3637528.3671575","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:04:19Z","timestamp":1750291459000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3637528.3671575"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,24]]},"references-count":25,"alternative-id":["10.1145\/3637528.3671575","10.1145\/3637528"],"URL":"https:\/\/doi.org\/10.1145\/3637528.3671575","relation":{},"subject":[],"published":{"date-parts":[[2024,8,24]]},"assertion":[{"value":"2024-08-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}