{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:01:11Z","timestamp":1755907271693,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":64,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,5,13]],"date-time":"2025-05-13T00:00:00Z","timestamp":1747094400000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS2125201, IIS2203097"],"award-info":[{"award-number":["IIS2125201, IIS2203097"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Intelligence Advanced Research Projects Activity"},{"DOI":"10.13039\/501100006374","name":"Office of the Director of National Intelligence","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Science Foundation, Faculty Early Career Development Program","award":["IIS2142739"],"award-info":[{"award-number":["IIS2142739"]}]},{"DOI":"10.13039\/501100006374","name":"Defense Advanced Research Projects Agency","doi-asserted-by":"publisher","award":["HR001120C0124"],"award-info":[{"award-number":["HR001120C0124"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Alfred P.~Sloan Foundation Fellowship"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,13]]},"DOI":"10.1145\/3589334.3645623","type":"proceedings-article","created":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T07:08:13Z","timestamp":1715152093000},"page":"2226-2237","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["KGQuiz: Evaluating the Generalization of Encoded Knowledge in Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6261-4583","authenticated-orcid":false,"given":"Yuyang","family":"Bai","sequence":"first","affiliation":[{"name":"Xi'an Jiaotong University, Xi'an, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4133-1987","authenticated-orcid":false,"given":"Shangbin","family":"Feng","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0465-0098","authenticated-orcid":false,"given":"Vidhisha","family":"Balachandran","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8230-6238","authenticated-orcid":false,"given":"Zhaoxuan","family":"Tan","sequence":"additional","affiliation":[{"name":"University of Notre Dame, Notre Dame, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4615-5481","authenticated-orcid":false,"given":"Shiqi","family":"Lou","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University, Xi'an, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6383-0307","authenticated-orcid":false,"given":"Tianxing","family":"He","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4634-7128","authenticated-orcid":false,"given":"Yulia","family":"Tsvetkov","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,5,13]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.527"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.667"},{"key":"e_1_3_2_2_3_1","volume-title":"Multimodal Evaluation of ChatGPT on Reasoning, Hallucination, and Interactivity. ArXiv abs\/2302.04023","author":"Bang Yejin","year":"2023","unstructured":"Yejin Bang, Samuel Cahyawijaya, Nayeon Lee, Wenliang Dai, Dan Su, Bryan Wilie, Holy Lovenia, Ziwei Ji, Tiezheng Yu, Willy Chung, Quyet V. Do, Yan Xu, and Pascale Fung. 2023. A Multitask, Multilingual, Multimodal Evaluation of ChatGPT on Reasoning, Hallucination, and Interactivity. ArXiv abs\/2302.04023 (2023)."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkh061"},{"key":"e_1_3_2_2_5_1","volume-title":"Proceedings of the 26th International Conference on Neural Information Processing Systems -","volume":"2","author":"Bordes Antoine","year":"2013","unstructured":"Antoine Bordes, Nicolas Usunier, Alberto Garcia-Dur\u00e1n, Jason Weston, and Oksana Yakhnenko. 2013. Translating Embeddings for Modeling Multi-Relational Data. In Proceedings of the 26th International Conference on Neural Information Processing Systems - Volume 2 (Lake Tahoe, Nevada) (NIPS'13). Curran Associates Inc., Red Hook, NY, USA, 2787--2795."},{"key":"e_1_3_2_2_6_1","volume-title":"PURR: Efficiently Editing Language Model Hallucinations by Denoising Language Model Corruptions. arXiv preprint arXiv:2305.14908","author":"Chen Anthony","year":"2023","unstructured":"Anthony Chen, Panupong Pasupat, Sameer Singh, Hongrae Lee, and Kelvin Guu. 2023. PURR: Efficiently Editing Language Model Hallucinations by Denoising Language Model Corruptions. arXiv preprint arXiv:2305.14908 (2023)."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.146"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.550"},{"key":"e_1_3_2_2_9_1","volume-title":"FELM: Benchmarking Factuality Evaluation of Large Language Models. arXiv:2310.00741 [cs.CL]","author":"Chen Shiqi","year":"2023","unstructured":"Shiqi Chen, Yiran Zhao, Jinghan Zhang, I-Chun Chern, Siyang Gao, Pengfei Liu, and Junxian He. 2023. FELM: Benchmarking Factuality Evaluation of Large Language Models. arXiv:2310.00741 [cs.CL]"},{"key":"e_1_3_2_2_10_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","volume":"1","author":"Clark Christopher","year":"2019","unstructured":"Christopher Clark, Kenton Lee, Ming-Wei Chang, Tom Kwiatkowski, Michael Collins, and Kristina Toutanova. 2019. BoolQ: Exploring the Surprising Difficulty of Natural Yes\/No Questions. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). 2924--2936."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00459"},{"key":"e_1_3_2_2_12_1","volume-title":"Wizard of Wikipedia: Knowledge-Powered Conversational Agents. In International Conference on Learning Representations. https:\/\/openreview.net\/ forum?id=r1l73iRqKm","author":"Dinan Emily","year":"2019","unstructured":"Emily Dinan, Stephen Roller, Kurt Shuster, Angela Fan, Michael Auli, and Jason Weston. 2019. Wizard of Wikipedia: Knowledge-Powered Conversational Agents. In International Conference on Learning Representations. https:\/\/openreview.net\/ forum?id=r1l73iRqKm"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.26"},{"key":"e_1_3_2_2_14_1","unstructured":"Shangbin Feng Weijia Shi Yuyang Bai Vidhisha Balachandran Tianxing He and Yulia Tsvetkov. 2023. CooK: Empowering General-Purpose Language Models with Modular and Collaborative Knowledge. arXiv:2305.09955 [cs.CL]"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.99"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0031619"},{"key":"e_1_3_2_2_17_1","volume-title":"Junyi Jessy Li, and Greg Durrett","author":"Goyal Tanya","year":"2023","unstructured":"Tanya Goyal, Junyi Jessy Li, and Greg Durrett. 2023. News Summarization and Evaluation in the Era of GPT-3. arXiv:2209.12356 [cs.CL]"},{"key":"e_1_3_2_2_18_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning (ICML'20)","author":"Guu Kelvin","year":"2020","unstructured":"Kelvin Guu, Kenton Lee, Zora Tung, Panupong Pasupat, and Ming-Wei Chang. 2020. REALM: Retrieval-Augmented Language Model Pre-Training. In Proceedings of the 37th International Conference on Machine Learning (ICML'20). JMLR.org, Article 368, 10 pages."},{"key":"e_1_3_2_2_19_1","unstructured":"Tianxing He Kyunghyun Cho and James Glass. 2021. An Empirical Study on Few-shot Knowledge Probing for Pretrained Language Models. arXiv:2109.02772 [cs.AI]"},{"key":"e_1_3_2_2_20_1","volume-title":"Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=d7KBjmI3GmQ","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=d7KBjmI3GmQ"},{"key":"e_1_3_2_2_21_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR)","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. Proceedings of the International Conference on Learning Representations (ICLR) (2021)."},{"key":"e_1_3_2_2_22_1","unstructured":"Evan Hernandez Belinda Z. Li and Jacob Andreas. 2023. Inspecting and Editing Knowledge Representations in Language Models. arXiv:2304.00740 [cs.CL]"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17--1147"},{"key":"e_1_3_2_2_25_1","volume-title":"Exploiting asymmetry for synthetic training data generation: Synthie and the case of information extraction. arXiv preprint arXiv:2303.04132","author":"Josifoski Martin","year":"2023","unstructured":"Martin Josifoski, Marija Sakota, Maxime Peyrard, and RobertWest. 2023. Exploiting asymmetry for synthetic training data generation: Synthie and the case of information extraction. arXiv preprint arXiv:2303.04132 (2023)."},{"volume-title":"Scott Johnston, Sheer El-Showk, Andy Jones","author":"Kadavath Saurav","key":"e_1_3_2_2_26_1","unstructured":"Saurav Kadavath, Tom Conerly, Amanda Askell, Tom Henighan, Dawn Drain, Ethan Perez, Nicholas Schiefer, Zac Hatfield-Dodds, Nova DasSarma, Eli Tran- Johnson, Scott Johnston, Sheer El-Showk, Andy Jones, Nelson Elhage, Tristan Hume, Anna Chen, Yuntao Bai, Sam Bowman, Stanislav Fort, Deep Ganguli, Danny Hernandez, Josh Jacobson, Jackson Kernion, Shauna Kravec, Liane Lovitt, Kamal Ndousse, Catherine Olsson, Sam Ringer, Dario Amodei, Tom Brown, Jack Clark, Nicholas Joseph, Ben Mann, Sam McCandlish, Chris Olah, and Jared Kaplan. 2022. Language Models (Mostly) Know What They Know. arXiv:2207.05221 [cs.CL]"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.307"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00276"},{"key":"e_1_3_2_2_29_1","volume-title":"Lin (Eds.)","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen-tau Yih, Tim Rockt\u00e4schel, Sebastian Riedel, and Douwe Kiela. 2020. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (Eds.), Vol. 33. Curran Associates, Inc., 9459--9474. https:\/\/proceedings.neurips.cc\/ paper_files\/paper\/2020\/file\/6b493230205f780e1bc26945df7481e5-Paper.pdf"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2020.2981314"},{"key":"e_1_3_2_2_31_1","volume-title":"Self-Prompting Large Language Models for Open-Domain QA. ArXiv abs\/2212.08635","author":"Li Junlong","year":"2022","unstructured":"Junlong Li, Zhuosheng Zhang, and Hai Zhao. 2022. Self-Prompting Large Language Models for Open-Domain QA. ArXiv abs\/2212.08635 (2022). https: \/\/api.semanticscholar.org\/CorpusID:254823646"},{"key":"e_1_3_2_2_32_1","volume-title":"Self-Checker: Plug-and-Play Modules for Fact-Checking with Large Language Models. ArXiv abs\/2305.14623","author":"Li Miaoran","year":"2023","unstructured":"Miaoran Li, Baolin Peng, and Zhu Zhang. 2023. Self-Checker: Plug-and-Play Modules for Fact-Checking with Large Language Models. ArXiv abs\/2305.14623 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258865801"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19--1282"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.173"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"crossref","unstructured":"Yixin Liu Alexander R. Fabbri Pengfei Liu Dragomir Radev and Arman Cohan. 2023. On Learning to Summarize with Large Language Models as References. arXiv:2305.14239 [cs.CL]","DOI":"10.18653\/v1\/2024.naacl-long.478"},{"volume-title":"Conference on Innovative Data Systems Research.","author":"Mahdisoltani Farzaneh","key":"e_1_3_2_2_36_1","unstructured":"Farzaneh Mahdisoltani, Joanna Asia Biega, and Fabian M. Suchanek. 2015. YAGO3: A Knowledge Base from Multilingual Wikipedias. In Conference on Innovative Data Systems Research."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"crossref","unstructured":"Alex Mallen Akari Asai Victor Zhong Rajarshi Das Daniel Khashabi and Hannaneh Hajishirzi. 2023. When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories. arXiv:2212.10511 [cs.CL]","DOI":"10.18653\/v1\/2023.acl-long.546"},{"key":"e_1_3_2_2_38_1","volume-title":"SelfCheck- GPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models. ArXiv abs\/2303.08896","author":"Manakul Potsawee","year":"2023","unstructured":"Potsawee Manakul, Adian Liusie, and Mark John Francis Gales. 2023. SelfCheck- GPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models. ArXiv abs\/2303.08896 (2023). https:\/\/api.semanticscholar.org\/ CorpusID:257557820"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.329"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1076"},{"key":"e_1_3_2_2_41_1","volume-title":"FActScore: Finegrained Atomic Evaluation of Factual Precision in Long Form Text Generation. arXiv preprint arXiv:2305.14251","author":"Min Sewon","year":"2023","unstructured":"Sewon Min, Kalpesh Krishna, Xinxi Lyu, Mike Lewis,Wen-tau Yih, PangWei Koh, Mohit Iyyer, Luke Zettlemoyer, and Hannaneh Hajishirzi. 2023. FActScore: Finegrained Atomic Evaluation of Factual Precision in Long Form Text Generation. arXiv preprint arXiv:2305.14251 (2023)."},{"key":"e_1_3_2_2_42_1","volume-title":"Rethinking Language Models as Symbolic Knowledge Graphs. ArXiv abs\/2308.13676","author":"Mruthyunjaya Vishwas","year":"2023","unstructured":"Vishwas Mruthyunjaya, Pouya Pezeshkpour, Estevam Hruschka, and Nikita Bhutani. 2023. Rethinking Language Models as Symbolic Knowledge Graphs. ArXiv abs\/2308.13676 (2023). https:\/\/api.semanticscholar.org\/CorpusID: 261242776"},{"key":"e_1_3_2_2_43_1","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Gray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell PeterWelinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. In Advances in Neural Information Processing Systems Alice H. Oh Alekh Agarwal Danielle Belgrave and Kyunghyun Cho (Eds.). https:\/\/openreview.net\/forum?id=TG8KACxEON"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.383"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.200"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19--1250"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.338"},{"key":"e_1_3_2_2_48_1","volume-title":"Christopher Michael Rytting, and David Wingate","author":"Robinson Joshua","year":"2022","unstructured":"Joshua Robinson, Christopher Michael Rytting, and David Wingate. 2022. Leveraging Large Language Models for Multiple Choice Question Answering. arXiv preprint arXiv:2210.12353 (2022)."},{"key":"e_1_3_2_2_49_1","volume-title":"REPLUG: Retrieval-Augmented Black-Box Language Models. arXiv:2301.12652 [cs.CL]","author":"Shi Weijia","year":"2023","unstructured":"Weijia Shi, Sewon Min, Michihiro Yasunaga, Minjoon Seo, Rich James, Mike Lewis, Luke Zettlemoyer, and Wen tau Yih. 2023. REPLUG: Retrieval-Augmented Black-Box Language Models. arXiv:2301.12652 [cs.CL]"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.11164"},{"key":"e_1_3_2_2_51_1","volume-title":"Hanwen Zha, Yue Liu, and Xin Luna Dong.","author":"Sun Kai","year":"2023","unstructured":"Kai Sun, Yifan Ethan Xu, Hanwen Zha, Yue Liu, and Xin Luna Dong. 2023. Headto- Tail: How Knowledgeable are Large Language Models (LLM)? A.K.A. Will LLMs Replace Knowledge Graphs? arXiv:2308.10168 [cs.CL]"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.388"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19--1421"},{"key":"e_1_3_2_2_54_1","volume-title":"Hashimoto","author":"Taori Rohan","year":"2023","unstructured":"Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. 2023. Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_ alpaca."},{"key":"e_1_3_2_2_55_1","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar Aurelien Rodriguez Armand Joulin Edouard Grave and Guillaume Lample. 2023. LLaMA: Open and Efficient Foundation Language Models. arXiv:2302.13971 [cs.CL]"},{"key":"e_1_3_2_2_56_1","unstructured":"BenWang and Aran Komatsuzaki. 2021. GPT-J-6B: A 6 Billion Parameter Autoregressive Language Model. https:\/\/github.com\/kingoflolz\/mesh-transformer-jax."},{"key":"e_1_3_2_2_57_1","unstructured":"Jian Xie Kai Zhang Jiangjie Chen Renze Lou and Yu Su. 2023. Adaptive Chameleon or Stubborn Sloth: Unraveling the Behavior of Large Language Models in Knowledge Clashes. arXiv:2305.13300 [cs.CL]"},{"key":"e_1_3_2_2_58_1","unstructured":"Michihiro Yasunaga Antoine Bosselut Hongyu Ren Xikun Zhang Christopher D. Manning Percy Liang and Jure Leskovec. 2022. Deep Bidirectional Language-Knowledge Graph Pretraining. In Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.45"},{"key":"e_1_3_2_2_60_1","unstructured":"Jifan Yu Xiaozhi Wang Shangqing Tu Shulin Cao Daniel Zhang-Li Xin Lv Hao Peng Zijun Yao Xiaohan Zhang Hanming Li Chunyang Li Zheyuan Zhang Yushi Bai Yantao Liu Amy Xin Nianyi Lin Kaifeng Yun Linlu Gong Jianhui Chen Zhili Wu Yunjia Qi Weikai Li Yong Guan Kaisheng Zeng Ji Qi Hailong Jin Jinxin Liu Yu Gu Yuan Yao Ning Ding Lei Hou Zhiyuan Liu Bin Xu Jie Tang and Juanzi Li. 2023. KoLA: Carefully Benchmarking World Knowledge of Large Language Models. arXiv:2306.09296 [cs.CL]"},{"key":"e_1_3_2_2_61_1","volume-title":"Todor Mihaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer.","author":"Zhang Susan","year":"2022","unstructured":"Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen, Christopher Dewan, Mona Diab, Xian Li, Xi Victoria Lin, Todor Mihaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer. 2022. OPT: Open Pre-trained Transformer Language Models. arXiv:2205.01068 [cs.CL]"},{"key":"e_1_3_2_2_62_1","volume-title":"Hashimoto","author":"Zhang Tianyi","year":"2023","unstructured":"Tianyi Zhang, Faisal Ladhak, Esin Durmus, Percy Liang, Kathleen McKeown, and Tatsunori B. Hashimoto. 2023. Benchmarking Large Language Models for News Summarization. arXiv:2301.13848 [cs.CL]"},{"key":"e_1_3_2_2_63_1","volume-title":"GreaseLM: Graph REASoning Enhanced Language Models. In International Conference on Learning Representations.","author":"Zhang Xikun","year":"2021","unstructured":"Xikun Zhang, Antoine Bosselut, Michihiro Yasunaga, Hongyu Ren, Percy Liang, Christopher D Manning, and Jure Leskovec. 2021. GreaseLM: Graph REASoning Enhanced Language Models. In International Conference on Learning Representations."},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.147"}],"event":{"name":"WWW '24: The ACM Web Conference 2024","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Singapore Singapore","acronym":"WWW '24"},"container-title":["Proceedings of the ACM Web Conference 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645623","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3589334.3645623","content-type":"text\/html","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589334.3645623","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589334.3645623","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T00:34:05Z","timestamp":1755822845000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645623"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,13]]},"references-count":64,"alternative-id":["10.1145\/3589334.3645623","10.1145\/3589334"],"URL":"https:\/\/doi.org\/10.1145\/3589334.3645623","relation":{},"subject":[],"published":{"date-parts":[[2024,5,13]]},"assertion":[{"value":"2024-05-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}