{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,28]],"date-time":"2025-03-28T07:42:14Z","timestamp":1743147734201,"version":"3.40.3"},"publisher-location":"Singapore","reference-count":47,"publisher":"Springer Nature Singapore","isbn-type":[{"type":"print","value":"9789819751303"},{"type":"electronic","value":"9789819751310"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-981-97-5131-0_41","type":"book-chapter","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T23:02:31Z","timestamp":1720738951000},"page":"482-495","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Benchmarking Biomedical Relation Knowledge in\u00a0Large Language Models"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-4844-3605","authenticated-orcid":false,"given":"Fenghui","family":"Zhang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0736-4512","authenticated-orcid":false,"given":"Kuo","family":"Yang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7856-7560","authenticated-orcid":false,"given":"Chenqian","family":"Zhao","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2252-1913","authenticated-orcid":false,"given":"Haixu","family":"Li","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1414-9354","authenticated-orcid":false,"given":"Xin","family":"Dong","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3654-7836","authenticated-orcid":false,"given":"Haoyu","family":"Tian","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4713-3594","authenticated-orcid":false,"given":"Xuezhong","family":"Zhou","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,7,12]]},"reference":[{"issue":"1","key":"41_CR1","doi-asserted-by":"publisher","first-page":"e48291","DOI":"10.2196\/48291","volume":"9","author":"A Abd-Alrazaq","year":"2023","unstructured":"Abd-Alrazaq, A., et al.: Large language models in medical education: opportunities, challenges, and future directions. JMIR Med. Educ. 9(1), e48291 (2023)","journal-title":"JMIR Med. Educ."},{"key":"41_CR2","doi-asserted-by":"crossref","unstructured":"Agrawal, G., Kumarage, T., Alghamdi, Z., Liu, H.: Can knowledge graphs reduce hallucinations in LLMs?: a survey. arXiv arXiv:2311.07914 (2024)","DOI":"10.18653\/v1\/2024.naacl-long.219"},{"key":"41_CR3","doi-asserted-by":"crossref","unstructured":"Bang, Y., et al.: A multitask, multilingual, multimodal evaluation of ChatGPT on reasoning, hallucination, and interactivity. arXiv arXiv:2302.04023 (2023)","DOI":"10.18653\/v1\/2023.ijcnlp-main.45"},{"issue":"Suppl\u20131","key":"41_CR4","doi-asserted-by":"publisher","first-page":"D267","DOI":"10.1093\/nar\/gkh061","volume":"32","author":"O Bodenreider","year":"2004","unstructured":"Bodenreider, O.: The Unified Medical Language System (UMLS): integrating biomedical terminology. Nucleic Acids Res. 32(Suppl\u20131), D267\u2013D270 (2004)","journal-title":"Nucleic Acids Res."},{"key":"41_CR5","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., et al.: Language models are few-shot learners. Adv. Neural. Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"41_CR6","unstructured":"Chiang, W.L., et al.: Vicuna: an open-source chatbot impressing GPT-4 with 90%* ChatGPT quality, March 2023"},{"key":"41_CR7","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Burstein, J., Doran, C., Solorio, T. (eds.) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), Minneapolis, Minnesota, June 2019, pp. 4171\u20134186. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/N19-1423","DOI":"10.18653\/v1\/N19-1423"},{"key":"41_CR8","doi-asserted-by":"crossref","unstructured":"Du, Z., et al.: GLM: general language model pretraining with autoregressive blank infilling. arXiv arXiv:2103.10360 (2022)","DOI":"10.18653\/v1\/2022.acl-long.26"},{"key":"41_CR9","unstructured":"Fei, Z., et al.: LawBench: benchmarking legal knowledge of large language models. arXiv arXiv:2309.16289 (2023)"},{"key":"41_CR10","unstructured":"Guan, X., et al.: Mitigating large language model hallucinations via autonomous knowledge graph-based retrofitting. arXiv arXiv:2311.13314 (2023)"},{"key":"41_CR11","doi-asserted-by":"publisher","unstructured":"He, Z., et al.: MedEval: a multi-level, multi-task, and multi-domain medical benchmark for language model evaluation. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, Singapore, December 2023, pp. 8725\u20138744. Association for Computational Linguistics (2023). https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.540","DOI":"10.18653\/v1\/2023.emnlp-main.540"},{"key":"41_CR12","doi-asserted-by":"crossref","unstructured":"Huang, L., et al.: A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions. arXiv arXiv:2311.05232 (2023)","DOI":"10.1145\/3703155"},{"issue":"12","key":"41_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3571730","volume":"55","author":"Z Ji","year":"2023","unstructured":"Ji, Z., et al.: Survey of hallucination in natural language generation. ACM Comput. Surv. 55(12), 1\u201338 (2023)","journal-title":"ACM Comput. Surv."},{"key":"41_CR14","doi-asserted-by":"publisher","first-page":"423","DOI":"10.1162\/tacl_a_00324","volume":"8","author":"Z Jiang","year":"2020","unstructured":"Jiang, Z., Xu, F.F., Araki, J., Neubig, G.: How can we know what language models know? Trans. Assoc. Computat. Linguist. 8, 423\u2013438 (2020)","journal-title":"Trans. Assoc. Computat. Linguist."},{"key":"41_CR15","unstructured":"Jin, H., Chen, S., Wu, M., Zhu, K.Q.: PsyEval: a comprehensive large language model evaluation benchmark for mental health. arXiv arXiv:2311.09189 (2023)"},{"key":"41_CR16","doi-asserted-by":"publisher","unstructured":"Kassner, N., Sch\u00fctze, H.: Negated and misprimed probes for pretrained language models: birds can talk, but cannot fly. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7811\u20137818. Association for Computational Linguistics, Online, July 2020. https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.698","DOI":"10.18653\/v1\/2020.acl-main.698"},{"issue":"D1","key":"41_CR17","doi-asserted-by":"publisher","first-page":"D1075","DOI":"10.1093\/nar\/gkv1075","volume":"44","author":"M Kuhn","year":"2016","unstructured":"Kuhn, M., Letunic, I., Jensen, L.J., Bork, P.: The SIDER database of drugs and side effects. Nucleic Acids Res. 44(D1), D1075\u2013D1079 (2016)","journal-title":"Nucleic Acids Res."},{"key":"41_CR18","unstructured":"Liang, P., et\u00a0al.: Holistic evaluation of language models. arXiv preprint arXiv:2211.09110 (2022)"},{"issue":"3","key":"41_CR19","first-page":"265","volume":"88","author":"CE Lipscomb","year":"2000","unstructured":"Lipscomb, C.E.: Medical subject headings (MESH). Bull. Med. Libr. Assoc. 88(3), 265 (2000)","journal-title":"Bull. Med. Libr. Assoc."},{"issue":"W1","key":"41_CR20","doi-asserted-by":"publisher","first-page":"W137","DOI":"10.1093\/nar\/gku412","volume":"42","author":"CC Liu","year":"2014","unstructured":"Liu, C.C., et al.: DiseaseConnect: a comprehensive web server for mechanism-based disease-disease connections. Nucleic Acids Res. 42(W1), W137\u2013W146 (2014)","journal-title":"Nucleic Acids Res."},{"key":"41_CR21","unstructured":"Luo, J., Li, T., Wu, D., Jenkin, M., Liu, S., Dudek, G.: Hallucination detection and hallucination mitigation: an investigation. arXiv arXiv:2401.08358 (2024)"},{"key":"41_CR22","doi-asserted-by":"publisher","unstructured":"Luo, L., Vu, T., Phung, D., Haf, R.: Systematic assessment of factual knowledge in large language models. In: Findings of the Association for Computational Linguistics, EMNLP 2023, pp. 13272\u201313286. Association for Computational Linguistics, Singapore, December 2023. https:\/\/doi.org\/10.18653\/v1\/2023.findings-emnlp.885","DOI":"10.18653\/v1\/2023.findings-emnlp.885"},{"key":"41_CR23","doi-asserted-by":"crossref","unstructured":"Omar, R., Mangukiya, O., Kalnis, P., Mansour, E.: ChatGPT versus traditional question answering for knowledge graphs: current status and future directions towards knowledge graph chatbots. arXiv arXiv:2302.06466 (2023)","DOI":"10.1145\/3588911"},{"key":"41_CR24","unstructured":"OpenAI: GPT-4 technical report. arXiv arXiv:2303.08774 (2024)"},{"key":"41_CR25","doi-asserted-by":"publisher","unstructured":"Petroni, F., et al.: Language models as knowledge bases? In: Inui, K., Jiang, J., Ng, V., Wan, X. (eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 2463\u20132473. Association for Computational Linguistics, Hong Kong, China, November 2019. https:\/\/doi.org\/10.18653\/v1\/D19-1250","DOI":"10.18653\/v1\/D19-1250"},{"issue":"D1","key":"41_CR26","doi-asserted-by":"publisher","first-page":"D833","DOI":"10.1093\/nar\/gkw943","volume":"45","author":"J Pi\u00f1ero","year":"2016","unstructured":"Pi\u00f1ero, J., et al.: DisGeNET: a comprehensive platform integrating information on human disease-associated genes and variants. Nucleic Acids Res. 45(D1), D833\u2013D839 (2016)","journal-title":"Nucleic Acids Res."},{"issue":"D1","key":"41_CR27","doi-asserted-by":"publisher","first-page":"D877","DOI":"10.1093\/nar\/gkw1012","volume":"45","author":"N Rappaport","year":"2017","unstructured":"Rappaport, N., et al.: MalaCards: an amalgamated human disease compendium with diverse clinical and genetic annotation and structured search. Nucleic Acids Res. 45(D1), D877\u2013D887 (2017)","journal-title":"Nucleic Acids Res."},{"key":"41_CR28","doi-asserted-by":"crossref","unstructured":"Roberts, A., Raffel, C., Shazeer, N.: How much knowledge can you pack into the parameters of a language model? arXiv arXiv:2002.08910 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.437"},{"key":"41_CR29","doi-asserted-by":"publisher","unstructured":"Schaefer, M., et al.: Large language models are universal biomedical simulators. bioRxiv (2023). https:\/\/doi.org\/10.1101\/2023.06.16.545235","DOI":"10.1101\/2023.06.16.545235"},{"issue":"7972","key":"41_CR30","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","volume":"620","author":"K Singhal","year":"2023","unstructured":"Singhal, K., et al.: Large language models encode clinical knowledge. nature 620(7972), 172\u2013180 (2023)","journal-title":"nature"},{"key":"41_CR31","unstructured":"Srivastava, A., et\u00a0al.: Beyond the imitation game: quantifying and extrapolating the capabilities of language models. arXiv preprint arXiv:2206.04615 (2022)"},{"key":"41_CR32","doi-asserted-by":"publisher","first-page":"743","DOI":"10.1162\/tacl_a_00342","volume":"8","author":"A Talmor","year":"2020","unstructured":"Talmor, A., Elazar, Y., Goldberg, Y., Berant, J.: oLMpics-on what language model pre-training captures. Trans. Assoc. Comput. Linguist. 8, 743\u2013758 (2020)","journal-title":"Trans. Assoc. Comput. Linguist."},{"issue":"6","key":"41_CR33","first-page":"7","volume":"3","author":"R Taori","year":"2023","unstructured":"Taori, R., et al.: Alpaca: a strong, replicable instruction-following model. Stanford Center Res. Found. Models 3(6), 7 (2023)","journal-title":"Stanford Center Res. Found. Models"},{"issue":"8","key":"41_CR34","doi-asserted-by":"publisher","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","volume":"29","author":"AJ Thirunavukarasu","year":"2023","unstructured":"Thirunavukarasu, A.J., Ting, D.S.J., Elangovan, K., Gutierrez, L., Tan, T.F., Ting, D.S.W.: Large language models in medicine. Nat. Med. 29(8), 1930\u20131940 (2023)","journal-title":"Nat. Med."},{"key":"41_CR35","unstructured":"Thoppilan, R., et\u00a0al.: LaMDA: language models for dialog applications. arXiv preprint arXiv:2201.08239 (2022)"},{"issue":"1","key":"41_CR36","doi-asserted-by":"publisher","first-page":"bbad493","DOI":"10.1093\/bib\/bbad493","volume":"25","author":"S Tian","year":"2024","unstructured":"Tian, S., et al.: Opportunities and challenges for ChatGPT and large language models in biomedicine and health. Brief. Bioinform. 25(1), bbad493 (2024)","journal-title":"Brief. Bioinform."},{"key":"41_CR37","unstructured":"Touvron, H., et al.: LLaMA: open and efficient foundation language models. arXiv arXiv:2302.13971 (2023)"},{"key":"41_CR38","first-page":"5998","volume":"30","author":"A Vaswani","year":"2017","unstructured":"Vaswani, A., et al.: Attention is all you need. Adv. Neural. Inf. Process. Syst. 30, 5998\u20136008 (2017)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"41_CR39","unstructured":"Wang, C., et al.: Evaluating Open-QA evaluation. Adv. Neural Inf. Process. Syst. 36 (2024)"},{"key":"41_CR40","first-page":"24824","volume":"35","author":"J Wei","year":"2022","unstructured":"Wei, J., et al.: Chain-of-thought prompting elicits reasoning in large language models. Adv. Neural. Inf. Process. Syst. 35, 24824\u201324837 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"issue":"D1","key":"41_CR41","doi-asserted-by":"publisher","first-page":"D1110","DOI":"10.1093\/nar\/gky1021","volume":"47","author":"Y Wu","year":"2019","unstructured":"Wu, Y., et al.: SymMap: an integrative database of traditional Chinese medicine enhanced by symptom mapping. Nucleic Acids Res. 47(D1), D1110\u2013D1117 (2019)","journal-title":"Nucleic Acids Res."},{"key":"41_CR42","unstructured":"Xie, J., Zhang, K., Chen, J., Lou, R., Su, Y.: Adaptive chameleon or stubborn sloth: unraveling the behavior of large language models in knowledge conflicts. arXiv preprint arXiv:2305.13300 (2023)"},{"issue":"4","key":"41_CR43","doi-asserted-by":"publisher","first-page":"1805","DOI":"10.1109\/JBHI.2018.2870728","volume":"23","author":"K Yang","year":"2018","unstructured":"Yang, K., et al.: HerGePred: heterogeneous network embedding representation for disease gene prediction. IEEE J. Biomed. Health Inform. 23(4), 1805\u20131815 (2018)","journal-title":"IEEE J. Biomed. Health Inform."},{"issue":"1","key":"41_CR44","doi-asserted-by":"publisher","first-page":"575","DOI":"10.1109\/TCBB.2020.3002771","volume":"19","author":"K Yang","year":"2020","unstructured":"Yang, K., et al.: PDGNet: predicting disease genes using a deep neural network with multi-view features. IEEE\/ACM Trans. Comput. Biol. Bioinf. 19(1), 575\u2013584 (2020)","journal-title":"IEEE\/ACM Trans. Comput. Biol. Bioinf."},{"key":"41_CR45","unstructured":"Yang, Z., Dai, Z., Yang, Y., Carbonell, J., Salakhutdinov, R.R., Le, Q.V.: XLNet: generalized autoregressive pretraining for language understanding. In: Wallach, H., et al. (eds.) Advances in Neural Information Processing Systems, vol.\u00a032. Curran Associates, Inc. (2019)"},{"key":"41_CR46","unstructured":"Zhao, W.X., et\u00a0al.: A survey of large language models. arXiv arXiv:2303.18223 (2023)"},{"key":"41_CR47","unstructured":"Zhu, K., et\u00a0al.: PromptBench: towards evaluating the robustness of large language models on adversarial prompts. arXiv preprint arXiv:2306.04528 (2023)"}],"container-title":["Lecture Notes in Computer Science","Bioinformatics Research and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-97-5131-0_41","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,23]],"date-time":"2024-11-23T23:35:14Z","timestamp":1732404914000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-97-5131-0_41"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9789819751303","9789819751310"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-981-97-5131-0_41","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"12 July 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ISBRA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Bioinformatics Research and Applications","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Kunming","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19 July 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21 July 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"isbra2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/bio.csu.edu.cn\/ISBRA2024\/ISBRA2024_Home.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}