{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,24]],"date-time":"2026-01-24T11:59:55Z","timestamp":1769255995920,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,21]],"date-time":"2024-10-21T00:00:00Z","timestamp":1729468800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,21]]},"DOI":"10.1145\/3627673.3679961","type":"proceedings-article","created":{"date-parts":[[2024,10,20]],"date-time":"2024-10-20T19:34:11Z","timestamp":1729452851000},"page":"3741-3746","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["The Factuality of Large Language Models in the Legal Domain"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1555-4415","authenticated-orcid":false,"given":"Rajaa","family":"El Hamdani","sequence":"first","affiliation":[{"name":"T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris, Palaiseau, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0468-0384","authenticated-orcid":false,"given":"Thomas","family":"Bonald","sequence":"additional","affiliation":[{"name":"T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris, Palaiseau, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8770-3969","authenticated-orcid":false,"given":"Fragkiskos D.","family":"Malliaros","sequence":"additional","affiliation":[{"name":"Universit\u00e9 Paris-Saclay, CentraleSup\u00e9lec, Inria, Gif-sur-Yvette, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0844-1391","authenticated-orcid":false,"given":"Nils","family":"Holzenberger","sequence":"additional","affiliation":[{"name":"T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris, Palaiseau, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7189-2796","authenticated-orcid":false,"given":"Fabian","family":"Suchanek","sequence":"additional","affiliation":[{"name":"T\u00e9l\u00e9com Paris, Institut Polytechnique de Paris, Palaiseau, France"}]}],"member":"320","published-online":{"date-parts":[[2024,10,21]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","unstructured":"Leonard Adolphs Shehzaad Dhuliawala and Thomas Hofmann. 2021. How to Query Language Models? https:\/\/doi.org\/10.48550\/arXiv.2108.01928 arXiv:2108.01928 [cs].","DOI":"10.48550\/arXiv.2108.01928"},{"key":"e_1_3_2_1_2_1","unstructured":"Badr AlKhamissi Millicent Li Asli Celikyilmaz Mona Diab and Marjan Ghazvininejad. 2022. A Review on Language Models as Knowledge Bases. http:\/\/arxiv.org\/abs\/2204.06031 arXiv:2204.06031 [cs]."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6239"},{"key":"e_1_3_2_1_4_1","unstructured":"Rishi Bommasani Drew A. Hudson Ehsan Adeli Russ Altman Simran Arora et al. 2021. On the Opportunities and Risks of Foundation Models. ArXiv (2021). https:\/\/crfm.stanford.edu\/assets\/report.pdf"},{"key":"e_1_3_2_1_5_1","volume-title":"Robert Dadashi, et al.","author":"Botev Aleksandar","year":"2024","unstructured":"Aleksandar Botev, Soham De, Samuel L Smith, Anushan Fernando, George-Cristian Muraru, Ruba Haroun, Leonard Berrada, Razvan Pascanu, Pier Giuseppe Sessa, Robert Dadashi, et al. 2024. RecurrentGemma: Moving Past Transformers for Efficient Open Language Models. arXiv preprint arXiv:2404.07839 (2024)."},{"key":"e_1_3_2_1_6_1","volume-title":"Advances in Neural Information Processing Systems","volume":"33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, and other. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems, Vol. 33. Curran Associates, Inc., 1877--1901. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00492"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3641289"},{"key":"e_1_3_2_1_9_1","volume-title":"Tianle Li, Dacheng Li, Hao Zhang, Banghua Zhu, Michael Jordan, Joseph E. Gonzalez, and Ion Stoica.","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios Nikolas Angelopoulos, Tianle Li, Dacheng Li, Hao Zhang, Banghua Zhu, Michael Jordan, Joseph E. Gonzalez, and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. arxiv: 2403.04132 [cs.AI]"},{"key":"e_1_3_2_1_10_1","volume-title":"Charles Sutton, Sebastian Gehrmann, et al.","author":"Chowdhery Aakanksha","year":"2023","unstructured":"Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, et al. 2023. Palm: Scaling language modeling with pathways. Journal of Machine Learning Research (2023)."},{"key":"e_1_3_2_1_11_1","volume-title":"Malik Boudiaf, Dominic Culver, Rui Melo, Caio Corro, Andre F. T. Martins, Fabrizio Esposito, Vera L\u00facia Raposo, Sofia Morgado, and Michael Desa.","author":"Colombo Pierre","year":"2024","unstructured":"Pierre Colombo, Telmo Pessoa Pires, Malik Boudiaf, Dominic Culver, Rui Melo, Caio Corro, Andre F. T. Martins, Fabrizio Esposito, Vera L\u00facia Raposo, Sofia Morgado, and Michael Desa. 2024. SaulLM-7B: A pioneering Large Language Model for Law. http:\/\/arxiv.org\/abs\/2403.03883 arXiv:2403.03883 [cs]."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1093\/jla\/laae003"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00410"},{"key":"e_1_3_2_1_14_1","volume-title":"LawBench: Benchmarking Legal Knowledge of Large Language Models. CoRR","author":"Fei Zhiwei","year":"2023","unstructured":"Zhiwei Fei, Xiaoyu Shen, Dawei Zhu, Fengzhe Zhou, Zhuo Han, Songyang Zhang, Kai Chen, Zongwen Shen, and Jidong Ge. 2023. LawBench: Benchmarking Legal Knowledge of Large Language Models. CoRR (2023)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.366"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4583531"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/320251.320253"},{"key":"e_1_3_2_1_19_1","unstructured":"Xuming Hu Junzhe Chen Xiaochuan Li Yufei Guo Lijie Wen Philip S Yu and Zhijiang Guo. 2024. TOWARDS UNDERSTANDING FACTUAL KNOWLEDGE OF LARGE LANGUAGE MODELS. (2024)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00324"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","unstructured":"Saurav Kadavath Tom Conerly Amanda Askell Tom Henighan Dawn Drain and other. 2022. Language Models (Mostly) Know What They Know. https:\/\/doi.org\/10.48550\/arXiv.2207.05221 arXiv:2207.05221 [cs].","DOI":"10.48550\/arXiv.2207.05221"},{"key":"e_1_3_2_1_23_1","volume-title":"KAMEL: Knowledge Analysis with Multitoken Entities in Language Models.. In AKBC.","author":"Kalo Jan-Christoph","year":"2022","unstructured":"Jan-Christoph Kalo and Leandra Fichtel. 2022. KAMEL: Knowledge Analysis with Multitoken Entities in Language Models.. In AKBC."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.3233\/sw-140134"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.557"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.eacl-main.213"},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the 18th Conference of the European","author":"Muhlgay Dor","year":"2024","unstructured":"Dor Muhlgay, Ori Ram, Inbal Magar, Yoav Levine, Nir Ratner, Yonatan Belinkov, Omri Abend, Kevin Leyton-Brown, Amnon Shashua, and Yoav Shoham. 2024. Generating Benchmarks for Factuality Evaluation of Language Models. In Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers), Yvette Graham and Matthew Purver (Eds.). Association for Computational Linguistics, St. Julian's, Malta, 49--66. https:\/\/aclanthology.org\/2024.eacl-long.4"},{"key":"e_1_3_2_1_33_1","volume-title":"Performance in the Courtroom: Automated Processing and Visualization of Appeal Court Decisions in France. In The 26Th Acm Sigkdd International Conference On Knowledge Discovery & Data Mining (Kdd","author":"Paul Boniol","year":"2020","unstructured":"Boniol Paul, Panagopoulos George, Xypolopoulos Christos, Hamdani Rajaa El, Amariles David Restrepo, and Vazirgianni Michalis. 2020. Performance in the Courtroom: Automated Processing and Visualization of Appeal Court Decisions in France. In The 26Th Acm Sigkdd International Conference On Knowledge Discovery & Data Mining (Kdd 2020)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_37_1","volume-title":"The authority of law: essays on law and morality reprinted ed.)","author":"Raz Joseph","unstructured":"Joseph Raz. 2002. The authority of law: essays on law and morality reprinted ed.). Oxford Univ. Press, Oxford."},{"key":"e_1_3_2_1_38_1","volume-title":"The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=98p5x51L5af","author":"Si Chenglei","year":"2023","unstructured":"Chenglei Si, Zhe Gan, Zhengyuan Yang, Shuohang Wang, Jianfeng Wang, Jordan Lee Boyd-Graber, and Lijuan Wang. 2023. Prompting GPT-3 To Be Reliable. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=98p5x51L5af"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-45072-3_1"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Fabian M. Suchanek Mehwish Alam Thomas Bonald Lihu Chen Pierre-Henri Paris and Jules Soria. 2024. YAGO 4.5: A Large and Clean Knowledge Base with a Rich Taxonomy. In SIGIR.","DOI":"10.1145\/3626772.3657876"},{"key":"e_1_3_2_1_41_1","volume-title":"Juliette Love, et al.","author":"Team Gemma","year":"2024","unstructured":"Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivi\u00e8re, Mihir Sanjay Kale, Juliette Love, et al. 2024. Gemma: Open models based on gemini research and technology. arXiv preprint arXiv:2403.08295 (2024)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi and other. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. https:\/\/doi.org\/10.48550\/arXiv.2307.09288 arXiv:2307.09288 [cs].","DOI":"10.48550\/arXiv.2307.09288"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2629489"},{"key":"e_1_3_2_1_44_1","unstructured":"Jason Wei Yi Tay Rishi Bommasani Colin Raffel Barret Zoph Sebastian Borgeaud Dani Yogatama Maarten Bosma Denny Zhou Donald Metzler et al. 2022. Emergent Abilities of Large Language Models. Transactions on Machine Learning Research (2022)."},{"key":"e_1_3_2_1_45_1","unstructured":"Jerry Wei Jason Wei Yi Tay Dustin Tran Albert Webson Yifeng Lu Xinyun Chen Hanxiao Liu Da Huang Denny Zhou et al. 2023. Larger language models do in-context learning differently. arXiv preprint arXiv:2303.03846 (2023)."},{"key":"e_1_3_2_1_46_1","volume-title":"Le","author":"Wei Jerry","year":"2024","unstructured":"Jerry Wei, Chengrun Yang, Xinying Song, Yifeng Lu, Nathan Hu, Jie Huang, Dustin Tran, Daiyi Peng, Ruibo Liu, Da Huang, Cosmo Du, and Quoc V. Le. 2024. Long-form factuality in large language models. http:\/\/arxiv.org\/abs\/2403.18802 arXiv:2403.18802 [cs]."},{"key":"e_1_3_2_1_47_1","volume-title":"Simon Razniewski, Fabian Suchanek, et al.","author":"Weikum Gerhard","year":"2021","unstructured":"Gerhard Weikum, Xin Luna Dong, Simon Razniewski, Fabian Suchanek, et al. 2021. Machine knowledge: Creation and curation of comprehensive knowledge bases. Foundations and Trends\u00ae in Databases, Vol. 10, 2--4 (2021), 108--490."},{"key":"e_1_3_2_1_48_1","volume-title":"KoLA: Carefully Benchmarking World Knowledge of Large Language Models. In The Twelfth International Conference on Learning Representations.","author":"Yu Jifan","year":"2024","unstructured":"Jifan Yu, Xiaozhi Wang, Shangqing Tu, Shulin Cao, Daniel Zhang-Li, Xin Lv, Hao Peng, Zijun Yao, Xiaohan Zhang, Hanming Li, Chunyang Li, Zheyuan Zhang, Yushi Bai, Yantao Liu, Amy Xin, Kaifeng Yun, Linlu GONG, Nianyi Lin, Jianhui Chen, Zhili Wu, Yunjia Qi, Weikai Li, Yong Guan, Kaisheng Zeng, Ji Qi, Hailong Jin, Jinxin Liu, Yu Gu, Yuan Yao, Ning Ding, Lei Hou, Zhiyuan Liu, Xu Bin, Jie Tang, and Juanzi Li. 2024. KoLA: Carefully Benchmarking World Knowledge of Large Language Models. In The Twelfth International Conference on Learning Representations."}],"event":{"name":"CIKM '24: The 33rd ACM International Conference on Information and Knowledge Management","location":"Boise ID USA","acronym":"CIKM '24","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 33rd ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627673.3679961","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3627673.3679961","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:58:09Z","timestamp":1750294689000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627673.3679961"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,21]]},"references-count":47,"alternative-id":["10.1145\/3627673.3679961","10.1145\/3627673"],"URL":"https:\/\/doi.org\/10.1145\/3627673.3679961","relation":{},"subject":[],"published":{"date-parts":[[2024,10,21]]},"assertion":[{"value":"2024-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}