{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T11:08:18Z","timestamp":1775560098495,"version":"3.50.1"},"reference-count":52,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T00:00:00Z","timestamp":1775520000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T00:00:00Z","timestamp":1775520000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61562010"],"award-info":[{"award-number":["61562010"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Key Research and Development Program of China","award":["2023YFC3341205"],"award-info":[{"award-number":["2023YFC3341205"]}]},{"name":"Guizhou Provincial Major Scientific and Technological Program","award":["[2024]003"],"award-info":[{"award-number":["[2024]003"]}]},{"name":"Guizhou Provincial Program on Commercialization of Scientific and Technological Achievements","award":["[2025]008"],"award-info":[{"award-number":["[2025]008"]}]},{"name":"Guizhou Provincial Program on Commercialization of Scientific and Technological Achievements","award":["[2023]010"],"award-info":[{"award-number":["[2023]010"]}]},{"name":"Research Projects of the Science and Technology Plan of Guizhou Province","award":["[2023]276"],"award-info":[{"award-number":["[2023]276"]}]},{"name":"Research Projects of the Science and Technology Plan of Guizhou Province","award":["[2022]261"],"award-info":[{"award-number":["[2022]261"]}]},{"name":"Research Projects of the Science and Technology Plan of Guizhou Province","award":["[2022]271"],"award-info":[{"award-number":["[2022]271"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Health Inf Sci Syst"],"DOI":"10.1007\/s13755-026-00448-9","type":"journal-article","created":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T10:21:07Z","timestamp":1775557267000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["UMEval: a unified framework for explainable medical term semantic evaluation with large language models"],"prefix":"10.1007","volume":"14","author":[{"given":"Shuyu","family":"Liu","sequence":"first","affiliation":[]},{"given":"Linkun","family":"Feng","sequence":"additional","affiliation":[]},{"given":"Youwei","family":"Luo","sequence":"additional","affiliation":[]},{"given":"Bijiang","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Panfeng","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Xiaohua","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Huarong","family":"Xu","sequence":"additional","affiliation":[]},{"given":"Dan","family":"Ma","sequence":"additional","affiliation":[]},{"given":"Mei","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Cen","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Hui","family":"Li","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7661-3917","authenticated-orcid":false,"given":"Yanhao","family":"Wang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2026,4,7]]},"reference":[{"issue":"10","key":"448_CR1","doi-asserted-by":"publisher","first-page":"1538","DOI":"10.1093\/jamia\/ocaa136","volume":"27","author":"Y Mao","year":"2020","unstructured":"Mao Y, Fung KW. Use of word and graph embedding to measure semantic relatedness between Unified Medical Language System concepts. J Am Med Inform Assoc. 2020;27(10):1538\u201346.","journal-title":"J Am Med Inform Assoc"},{"key":"448_CR2","first-page":"572","volume":"2010","author":"S Pakhomov","year":"2010","unstructured":"Pakhomov S, McInnes B, Adam T, Liu Y, Pedersen T, Melton GB. Semantic similarity and relatedness between clinical terms: an experimental study. AMIA Ann Symp Proc. 2010;2010:572\u20136.","journal-title":"AMIA Ann Symp Proc"},{"issue":"6","key":"448_CR3","doi-asserted-by":"publisher","first-page":"2514460","DOI":"10.1001\/jamanetworkopen.2025.14460","volume":"8","author":"MF Griot","year":"2025","unstructured":"Griot MF, Walker GA. A patient-in-the-loop approach to artificial intelligence in medicine. JAMA Netw Open. 2025;8(6):2514460.","journal-title":"JAMA Netw Open"},{"issue":"6","key":"448_CR4","doi-asserted-by":"publisher","first-page":"2514452","DOI":"10.1001\/jamanetworkopen.2025.14452","volume":"8","author":"F Busch","year":"2025","unstructured":"Busch F, Hoffmann L, Xu L. Multinational attitudes toward AI in health care and diagnostics among hospital patients. JAMA Netw Open. 2025;8(6):2514452.","journal-title":"JAMA Netw Open"},{"key":"448_CR5","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijmedinf.2022.104884","volume":"168","author":"C-F Liu","year":"2022","unstructured":"Liu C-F, Chen Z-C, Kuo S-C, Lin T-C. Does AI explainability affect physicians\u2019 intention to use AI? Int J Med Inform. 2022;168:104884.","journal-title":"Int J Med Inform"},{"key":"448_CR6","first-page":"431","volume":"2009","author":"BT McInnes","year":"2009","unstructured":"McInnes BT, Pedersen T, Pakhomov SVS. UMLS-interface and UMLS-similarity: open source software for measuring paths and semantic similarity. AMIA Ann Symp Proc. 2009;2009:431\u20135.","journal-title":"AMIA Ann Symp Proc"},{"issue":"1","key":"448_CR7","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1109\/21.24528","volume":"19","author":"R Rada","year":"1989","unstructured":"Rada R, Mili H, Bicknell E, Blettner M. Development and application of a metric on semantic nets. IEEE Trans Syst Man Cybern. 1989;19(1):17\u201330.","journal-title":"IEEE Trans Syst Man Cybern"},{"issue":"3","key":"448_CR8","doi-asserted-by":"publisher","first-page":"288","DOI":"10.1016\/j.jbi.2006.06.004","volume":"40","author":"T Pedersen","year":"2007","unstructured":"Pedersen T, Pakhomov SVS, Patwardhan S, Chute CG. Measures of semantic similarity and relatedness in the biomedical domain. J Biomed Inform. 2007;40(3):288\u201399.","journal-title":"J Biomed Inform"},{"issue":"7\u20138","key":"448_CR9","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1002\/cfg.499","volume":"6","author":"R Stevens","year":"2005","unstructured":"Stevens R, McEntire R, Lord P, Butler JA. Evaluating GO-based semantic similarity measures. Comp Funct Genomics. 2005;6(7\u20138):370\u20132.","journal-title":"Comp Funct Genomics"},{"issue":"5","key":"448_CR10","doi-asserted-by":"publisher","first-page":"749","DOI":"10.1016\/j.jbi.2011.03.013","volume":"44","author":"D S\u00e1nchez","year":"2011","unstructured":"S\u00e1nchez D, Batet M. Semantic similarity estimation in the biomedical domain: an ontology-based information-theoretic perspective. J Biomed Inform. 2011;44(5):749\u201359.","journal-title":"J Biomed Inform"},{"key":"448_CR11","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2020.103581","volume":"111","author":"S Jiang","year":"2020","unstructured":"Jiang S, Wu W, Tomita N, Ganoe C, Hassanpour S. Multi-ontology refined embeddings (MORE): a hybrid multi-ontology and corpus-based semantic representation model for biomedical concepts. J Biomed Inform. 2020;111:103581.","journal-title":"J Biomed Inform"},{"key":"448_CR12","first-page":"3111","volume":"26","author":"T Mikolov","year":"2013","unstructured":"Mikolov T, Sutskever I, Chen K, Corrado GS, Dean J. Distributed representations of words and phrases and their compositionality. Adv Neural Inf Process Syst. 2013;26:3111\u20139.","journal-title":"Adv Neural Inf Process Syst"},{"key":"448_CR13","unstructured":"Devlin J, Chang M-W, Lee K, Toutanova K. BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 conference of the north american chapter of the association for computational linguistics: human language technologies, vol 1 (long and short papers). 2019. p. 4171\u201386"},{"issue":"23","key":"448_CR14","doi-asserted-by":"publisher","first-page":"3635","DOI":"10.1093\/bioinformatics\/btw529","volume":"32","author":"SVS Pakhomov","year":"2016","unstructured":"Pakhomov SVS, Finley G, McEwan R, Wang Y, Melton GB. Corpus domain effects on distributional semantic modeling of medical terms. Bioinformatics. 2016;32(23):3635\u201344.","journal-title":"Bioinformatics"},{"key":"448_CR15","doi-asserted-by":"publisher","first-page":"158","DOI":"10.18653\/v1\/W15-3820","volume":"15","author":"M Th","year":"2015","unstructured":"Th M, Sahu S, Anand A. Evaluating distributed word representations for capturing semantics of biomedical concepts. Proc BioNLP. 2015;15:158\u201363.","journal-title":"Proc BioNLP"},{"issue":"1","key":"448_CR16","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1038\/s41597-019-0055-0","volume":"6","author":"Y Zhang","year":"2019","unstructured":"Zhang Y, Chen Q, Yang Z, Lin H, Lu Z. BioWordVec, improving biomedical word embeddings with subword information and MeSH. Sci Data. 2019;6(1):52.","journal-title":"Sci Data"},{"key":"448_CR17","doi-asserted-by":"crossref","unstructured":"Peng Y, Yan S, Lu Z. Transfer learning in biomedical natural language processing: an evaluation of BERT and ELMo on ten benchmarking datasets. In: Proceedings of the 18th BioNLP workshop and shared task. 2019. p. 58\u201365","DOI":"10.18653\/v1\/W19-5006"},{"key":"448_CR18","doi-asserted-by":"crossref","unstructured":"Alsentzer E, Murphy J, Boag W, Weng W-H, Jindi D, Naumann T, McDermott M. Publicly available clinical BERT embeddings. In: Proceedings of the 2nd clinical natural language processing workshop. 2019. p. 72\u20138","DOI":"10.18653\/v1\/W19-1909"},{"issue":"4","key":"448_CR19","doi-asserted-by":"publisher","first-page":"1234","DOI":"10.1093\/bioinformatics\/btz682","volume":"36","author":"J Lee","year":"2020","unstructured":"Lee J, Yoon W, Kim S, Kim D, Kim S, So CH, et al. BioBERT: a pre-trained biomedical language representation model for biomedical text mining. Bioinformatics. 2020;36(4):1234\u201340.","journal-title":"Bioinformatics"},{"key":"448_CR20","doi-asserted-by":"publisher","DOI":"10.3389\/fmolb.2022.928530","volume":"9","author":"N Saeed","year":"2022","unstructured":"Saeed N, Naveed H. Medical terminology-based computing system: a lightweight post-processing solution for out-of-vocabulary multi-word terms. Front Mol Biosci. 2022;9:928530.","journal-title":"Front Mol Biosci"},{"key":"448_CR21","doi-asserted-by":"crossref","unstructured":"Sousa RT, Silva S, Pesquita C. The supervised semantic similarity toolkit. In: The semantic web: ESWC 2022 satellite events\u2014Hersonissos, Crete, Greece, May 2022;29-2, Proceedings. 2022. p. 42\u20136","DOI":"10.1007\/978-3-031-11609-4_8"},{"key":"448_CR22","first-page":"324","volume":"2024","author":"Y Liu","year":"2024","unstructured":"Liu Y, Melton GB, Zhang R. Exploring large language models for acronym, symbol sense disambiguation, and semantic similarity and relatedness assessment. AMIA Summits Transl Sci Proc. 2024;2024:324\u201333.","journal-title":"AMIA Summits Transl Sci Proc"},{"key":"448_CR23","first-page":"657","volume":"245","author":"Z Yu","year":"2017","unstructured":"Yu Z, Wallace BC, Johnson T, Cohen T. Retrofitting concept vector representations of medical concepts to improve estimates of semantic similarity and relatedness. Stud Health Technol Inform. 2017;245:657\u201361.","journal-title":"Stud Health Technol Inform"},{"key":"448_CR24","first-page":"295","volume":"25","author":"AL Beam","year":"2020","unstructured":"Beam AL, Kompa B, Schmaltz A, Fried I, Weber G, Palmer N, et al. Clinical concept embeddings learned from massive sources of multimodal medical data. Pac Symp Biocomput. 2020;25:295\u2013306.","journal-title":"Pac Symp Biocomput"},{"key":"448_CR25","unstructured":"Kalyan KS, Sangeetha S. A hybrid approach to measure semantic relatedness in biomedical concepts. 2021. arxiv:org\/abs\/2101.10196"},{"key":"448_CR26","doi-asserted-by":"crossref","unstructured":"Liu F, Shareghi E, Meng Z, Basaldella M, Collier N. Self-alignment pretraining for biomedical entity representations. In: Proceedings of the 2021 conference of the North American chapter of the association for computational linguistics: human language technologies. 2021. p. 4228\u201338","DOI":"10.18653\/v1\/2021.naacl-main.334"},{"key":"448_CR27","doi-asserted-by":"crossref","unstructured":"Michalopoulos G, Wang Y, Kaka H, Chen H, Wong A. UmlsBERT: Clinical domain knowledge augmentation of contextual embeddings using the Unified Medical Language System metathesaurus. In: Proceedings of the 2021 conference of the North American chapter of the association for computational linguistics: human language technologies. 2021. p. 1744\u201353","DOI":"10.18653\/v1\/2021.naacl-main.139"},{"key":"448_CR28","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2021.103983","volume":"126","author":"Z Yuan","year":"2022","unstructured":"Yuan Z, Zhao Z, Sun H, Li J, Wang F, Yu S. CODER: knowledge-infused cross-lingual medical term embedding for term normalization. J Biomed Inform. 2022;126:103983.","journal-title":"J Biomed Inform"},{"issue":"1","key":"448_CR29","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1186\/s12911-022-01850-5","volume":"22","author":"AK Chanda","year":"2022","unstructured":"Chanda AK, Bai T, Yang Z, Vucetic S. Improving medical term embeddings using UMLS Metathesaurus. BMC Med Inform Decis Mak. 2022;22(1):114.","journal-title":"BMC Med Inform Decis Mak"},{"issue":"9","key":"448_CR30","doi-asserted-by":"publisher","first-page":"1844","DOI":"10.1093\/jamia\/ocae029","volume":"31","author":"F Remy","year":"2024","unstructured":"Remy F, Demuynck K, Demeester T. BioLORD-2023: semantic textual representations fusing large language models and clinical knowledge graph insights. J Am Med Inform Assoc. 2024;31(9):1844\u201355.","journal-title":"J Am Med Inform Assoc"},{"key":"448_CR31","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2024.108076","volume":"170","author":"RT Sousa","year":"2024","unstructured":"Sousa RT, Silva S, Pesquita C. Explaining protein-protein interactions with knowledge graph-based semantic similarity. Comput Biol Med. 2024;170:108076.","journal-title":"Comput Biol Med"},{"issue":"20","key":"448_CR32","doi-asserted-by":"publisher","first-page":"3579","DOI":"10.1093\/bioinformatics\/btab252","volume":"37","author":"Y Li","year":"2021","unstructured":"Li Y, Keqi W, Wang G. Evaluating disease similarity based on gene network reconstruction and representation. Bioinformatics. 2021;37(20):3579\u201387.","journal-title":"Bioinformatics"},{"issue":"6","key":"448_CR33","doi-asserted-by":"publisher","first-page":"1069","DOI":"10.1093\/bib\/bbv011","volume":"16","author":"R Hoehndorf","year":"2015","unstructured":"Hoehndorf R, Schofield PN, Gkoutos GV. The role of ontologies in biological and biomedical research: a functional perspective. Brief Bioinform. 2015;16(6):1069\u201380.","journal-title":"Brief Bioinform"},{"key":"448_CR34","volume-title":"Medical and care compunetics 3","author":"L Bos","year":"2006","unstructured":"Bos L, Roa L, Yogesan K, O\u2019Connell B, Marsh A, Blobel B. Medical and care compunetics 3. Amsterdam: IOS Press; 2006."},{"issue":"1","key":"448_CR35","doi-asserted-by":"publisher","first-page":"118","DOI":"10.1016\/j.jbi.2010.09.002","volume":"44","author":"M Batet","year":"2011","unstructured":"Batet M, S\u00e1nchez D, Valls A. An ontology-based measure to compute semantic similarity in biomedicine. J Biomed Inform. 2011;44(1):118\u201325.","journal-title":"J Biomed Inform"},{"issue":"12","key":"448_CR36","doi-asserted-by":"publisher","first-page":"1895","DOI":"10.1093\/jamia\/ocad161","volume":"30","author":"A Newbury","year":"2023","unstructured":"Newbury A, Liu H, Idnay B, Weng C. The suitability of UMLS and SNOMED-CT for encoding outcome concepts. J Am Med Inform Assoc. 2023;30(12):1895\u2013903.","journal-title":"J Am Med Inform Assoc"},{"key":"448_CR37","doi-asserted-by":"crossref","unstructured":"Chiu B, Crichton G, Korhonen A, Pyysalo S. How to train good word embeddings for biomedical NLP. In: Proceedings of the 15th workshop on biomedical natural language processing. 2016. p. 166\u201374","DOI":"10.18653\/v1\/W16-2922"},{"issue":"1","key":"448_CR38","doi-asserted-by":"publisher","first-page":"100","DOI":"10.1038\/s41746-024-01081-0","volume":"7","author":"S Gilbert","year":"2024","unstructured":"Gilbert S, Kather JN, Hogan A. Augmented non-hallucinating large language models as medical information curators. npj Digit Med. 2024;7(1):100.","journal-title":"npj Digit Med"},{"issue":"12","key":"448_CR39","doi-asserted-by":"publisher","first-page":"1274","DOI":"10.1111\/medu.13102","volume":"50","author":"WE Hautz","year":"2016","unstructured":"Hautz WE, Krummrey G, Exadaktylos A, Hautz SC. Six degrees of separation: the small world of medical education. Med Educ. 2016;50(12):1274\u20139.","journal-title":"Med Educ"},{"key":"448_CR40","doi-asserted-by":"crossref","unstructured":"Guo Z, Xia L, Yu Y, Ao T, Huang C. LightRAG: simple and fast retrieval-augmented generation. In: Findings of the association for computational linguistics: EMNLP 2025. 2025. p. 10746\u201361","DOI":"10.18653\/v1\/2025.findings-emnlp.568"},{"key":"448_CR41","unstructured":"Mohan S, Li D. MedMentions: a large biomedical corpus annotated with UMLS concepts. 2019. arxiv:org\/abs\/1902.09476"},{"key":"448_CR42","unstructured":"Panahi M. OpenMed NER: open-source, domain-adapted state-of-the-art transformers for biomedical NER across 12 public datasets. 2025. arxiv:org\/abs\/2508.01630"},{"key":"448_CR43","unstructured":"DeepSeek-AI: DeepSeek-V3 technical report. 2024. arxiv:org\/abs\/2412.19437"},{"key":"448_CR44","unstructured":"Qwen Team: Qwen3: think deeper, act faster. 2025. https:\/\/qwenlm.github.io\/blog\/qwen3\/"},{"key":"448_CR45","unstructured":"Zhang Y, Li M, Long D, Zhang X, Lin H, Yang B, Xie P, Yang A, Liu D, Lin J, Huang F, Zhou J. Qwen3 embedding: advancing text embedding and reranking through foundation models. 2025. arxiv:org\/abs\/2506.05176"},{"key":"448_CR46","unstructured":"Pakhomov S. Semantic relatedness and similarity reference standards for medical terms. 2018. https:\/\/doi.org\/10.13020\/D6CX04"},{"key":"448_CR47","unstructured":"Holtzman A, Buys J, Du MF, Choi Y. The curious case of neural text degeneration. In: The eighth international conference on learning representations 2020. https:\/\/openreview.net\/forum?id=rygGQyrFvH"},{"key":"448_CR48","unstructured":"Wang B, Zhao H, Zhou H, Song L, Xu M, Cheng W, Zeng X, Zhang Y, Huo Y, Wang Z, et al. Baichuan-M1: pushing the medical capability of large Language models. 2025. arxiv:org\/abs\/2502.12671"},{"key":"448_CR49","unstructured":"Zhipu AI: GLM-4 series: Open Multilingual Multimodal Chat LMs. 2024. https:\/\/github.com\/zai-org\/GLM-4"},{"key":"448_CR50","unstructured":"Anthropic: Claude 3.7 Sonnet and Claude Code. 2025. https:\/\/www.anthropic.com\/news\/claude-3-7-sonnet"},{"key":"448_CR51","unstructured":"OpenAI: Introducing GPT-4.1 in the API. 2025. https:\/\/openai.com\/index\/gpt-4-1\/"},{"key":"448_CR52","unstructured":"OpenAI: Introducing GPT-5.2 2025. https:\/\/openai.com\/index\/introducing-gpt-5-2\/"}],"container-title":["Health Information Science and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s13755-026-00448-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s13755-026-00448-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s13755-026-00448-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T10:21:19Z","timestamp":1775557279000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s13755-026-00448-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,7]]},"references-count":52,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,12]]}},"alternative-id":["448"],"URL":"https:\/\/doi.org\/10.1007\/s13755-026-00448-9","relation":{},"ISSN":["2047-2501"],"issn-type":[{"value":"2047-2501","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,4,7]]},"assertion":[{"value":"28 August 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 March 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 April 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no conflict of interest to declare that are relevant to the content of this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"52"}}