{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T06:56:49Z","timestamp":1782802609497,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":20,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T00:00:00Z","timestamp":1768262400000},"content-version":"vor","delay-in-days":211,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2204926"],"award-info":[{"award-number":["2204926"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,16]]},"DOI":"10.1145\/3769126.3769245","type":"proceedings-article","created":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T14:50:37Z","timestamp":1768315837000},"page":"425-429","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["LLMs Provide Unstable Answers to Legal Questions"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8547-9417","authenticated-orcid":false,"given":"Andrew","family":"Blair-Stanek","sequence":"first","affiliation":[{"name":"Carey School of Law, University of Maryland, Baltimore, Maryland, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4328-4288","authenticated-orcid":false,"given":"Benjamin","family":"Van Durme","sequence":"additional","affiliation":[{"name":"Computer Science, Johns Hopkins University, Baltimore, Maryland, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,1,13]]},"reference":[{"key":"e_1_3_3_1_2_2","doi-asserted-by":"publisher","unstructured":"Berk Atil Alexa Chittams Liseng Fu Ferhan Ture Lixinyu Xu and Breck Baldwin. 2024. LLM Stability: A detailed analysis with some surprises. arXiv preprint (2024). 10.48550\/arXiv.2408.04667","DOI":"10.48550\/arXiv.2408.04667"},{"key":"e_1_3_3_1_3_2","unstructured":"Nicole Black. 2024. Artificial intelligence tools for brief writing and analysis are a small firm litigator\u2019s new best friend. ABA Journal (2024)."},{"key":"e_1_3_3_1_4_2","unstructured":"Robert\u00a0E Blackwell Jon Barry and Anthony\u00a0G Cohn. 2024. Towards Reproducible LLM Evaluation: Quantifying Uncertainty in LLM Benchmark Scores. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.03492 (2024)."},{"key":"e_1_3_3_1_5_2","unstructured":"Charles Corbi\u00e8re Nicolas Thome Avner Bar-Hen Matthieu Cord and Patrick P\u00e9rez. 2019. Addressing failure prediction by learning model confidence. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_3_1_6_2","unstructured":"American Bar Association Law\u00a0Practice Division. July 16 2024. AI Tools for Legal Work: Claude Gemini Copilot and More."},{"key":"e_1_3_3_1_7_2","volume-title":"Statistics with Confidence\u2014Confidence Intervals and Statistical Guidelines","author":"Gardner M.\u00a0J.","year":"1990","unstructured":"M.\u00a0J. Gardner and D.\u00a0G. Altman.1990. Statistics with Confidence\u2014Confidence Intervals and Statistical Guidelines. BMJ."},{"key":"e_1_3_3_1_8_2","unstructured":"Global Market Insights. 2025. Legal AI Market Size-By Deployment Model By Technology By Application By End Use 2025-2034."},{"key":"e_1_3_3_1_9_2","unstructured":"Judicial Inquiry and Review\u00a0Commission of\u00a0Virginia\u00a0v. Shull. 2007. 274 Va. 657."},{"key":"e_1_3_3_1_10_2","unstructured":"Heinrich Jiang Been Kim Melody Guan and Maya Gupta. 2018. To trust or not to trust a classifier. Advances in neural information processing systems 31 (2018)."},{"key":"e_1_3_3_1_11_2","unstructured":"Saurav Kadavath Tom Conerly Amanda Askell and et al.2022. Language Models (Mostly) Know What They Know. https:\/\/arxiv.org\/abs\/2207.05221"},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"publisher","unstructured":"Takeshi Kojima Shixiang\u00a0Shane Gu Machel Reid Yutaka Matsuo and Yusuke Iwasawa. 2022. Large Language Models are Zero-Shot Reasoners. CoRR abs\/2205.11916 (2022). 10.48550\/arXiv.2205.11916 arXiv:https:\/\/arXiv.org\/abs\/2205.11916","DOI":"10.48550\/arXiv.2205.11916"},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","unstructured":"Jinqi Lai Wensheng Gan Jiayang Wu Zhenlian Qi and Philip\u00a0S. Yu. 2024. Large language models in law: A survey. AI Open 5 (2024) 181\u2013196. 10.1016\/j.aiopen.2024.09.002","DOI":"10.1016\/j.aiopen.2024.09.002"},{"key":"e_1_3_3_1_14_2","unstructured":"Zhen Lin Shubhendu Trivedi and Jimeng Sun. 2024. Generating with Confidence: Uncertainty Quantification for Black-box Large Language Models. Transactions on Machine Learning Research (2024)."},{"key":"e_1_3_3_1_15_2","unstructured":"Junnan Liu Hongwei Liu Linchen Xiao Ziyi Wang Kuikun Liu Songyang Gao Wenwei Zhang Songyang Zhang and Kai Chen. 2024. Are Your LLMs Capable of Stable Reasoning? arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2412.13147 (2024)."},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416545"},{"key":"e_1_3_3_1_17_2","unstructured":"The President and Fellows of Harvard\u00a0University. 2024. Caselaw Access Project."},{"key":"e_1_3_3_1_18_2","doi-asserted-by":"crossref","unstructured":"Matthew Renze and Erhan Guven. 2024. The Effect of Sampling Temperature on Problem Solving in Large Language Models. EMNLP (2024).","DOI":"10.18653\/v1\/2024.findings-emnlp.432"},{"key":"e_1_3_3_1_19_2","unstructured":"Adam\u00a0M. Samaha. 2009. Randomization in Adjudication. William & Mary Law Review 51 (2009) 1\u201386."},{"key":"e_1_3_3_1_20_2","unstructured":"Central Hudson Gas & Electric\u00a0Corp. v.\u00a0Public Service Commission\u00a0of NY. 1980. 447 U.S. 557."},{"key":"e_1_3_3_1_21_2","unstructured":"Hannes Westermann Jaromir Savelka and Karim Benyekhlef. 2023. LLMediator: GPT-4 Assisted Online Dispute Resolution. Workshop on Artificial Intelligence for Access to Justice (2023). https:\/\/ceur-ws.org\/Vol-3435\/paper1.pdf"}],"event":{"name":"ICAIL 2025: 20th International Conference on Artificial Intelligence and Law","location":"Chicago , IL , USA","acronym":"ICAIL 2025"},"container-title":["Proceedings of the Twentieth International Conference on Artificial Intelligence and Law"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3769126.3769245","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3769126.3769245","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T15:50:27Z","timestamp":1768319427000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3769126.3769245"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,16]]},"references-count":20,"alternative-id":["10.1145\/3769126.3769245","10.1145\/3769126"],"URL":"https:\/\/doi.org\/10.1145\/3769126.3769245","relation":{},"subject":[],"published":{"date-parts":[[2025,6,16]]},"assertion":[{"value":"2026-01-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}