{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T00:29:09Z","timestamp":1765499349501,"version":"3.48.0"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,10]]},"DOI":"10.1145\/3746252.3761491","type":"proceedings-article","created":{"date-parts":[[2025,11,7]],"date-time":"2025-11-07T23:59:18Z","timestamp":1762559958000},"page":"6738-6742","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["JustEva: A Toolkit to Evaluate LLM Fairness in Legal Knowledge Inference"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-2874-6211","authenticated-orcid":false,"given":"Zongyue","family":"Xue","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China and Yale Law School, New Haven, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1253-4495","authenticated-orcid":false,"given":"Siyuan","family":"Zheng","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China and Shanghai Jiaotong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-0356-8722","authenticated-orcid":false,"given":"Shaochun","family":"Wang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7986-3692","authenticated-orcid":false,"given":"Yiran","family":"Hu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China and University of Waterloo, Waterloo, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-9635-6486","authenticated-orcid":false,"given":"Yuxin","family":"Yao","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-1768-3082","authenticated-orcid":false,"given":"Shengran","family":"Wang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-8766-8610","authenticated-orcid":false,"given":"Haitao","family":"Li","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5030-709X","authenticated-orcid":false,"given":"Qingyao","family":"Ai","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0140-4512","authenticated-orcid":false,"given":"Yiqun","family":"Liu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-8684-7977","authenticated-orcid":false,"given":"Yun","family":"Liu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-7539-4242","authenticated-orcid":false,"given":"Weixing","family":"Shen","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,11,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"American Bar Association. 2024. AI Tools for Legal Work: Claude Gemini Copilot and More. https:\/\/www.americanbar.org\/groups\/law_practice\/resources\/law-technology-today\/2024\/ai-tools-for-legal-work-claude-gemini-copilot-and-more\/ Accessed: 2025-05-25."},{"key":"e_1_3_2_1_2_1","volume-title":"Machine Bias: Risk Assessments in Criminal Sentencing. https:\/\/www.propublica.org\/article\/machine-bias-risk-assessments-in-criminal-sentencing. ProPublica, Accessed","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. 2016. Machine Bias: Risk Assessments in Criminal Sentencing. https:\/\/www.propublica.org\/article\/machine-bias-risk-assessments-in-criminal-sentencing. ProPublica, Accessed: 5 June 2025."},{"volume-title":"Artificial intelligence and legal analytics: new tools for law practice in the digital age","author":"Ashley Kevin D","key":"e_1_3_2_1_3_1","unstructured":"Kevin D Ashley. 2017. Artificial intelligence and legal analytics: new tools for law practice in the digital age. Cambridge University Press."},{"key":"e_1_3_2_1_4_1","first-page":"671","article-title":"Big data's disparate impact","volume":"104","author":"Barocas Solon","year":"2016","unstructured":"Solon Barocas and Andrew D Selbst. 2016. Big data's disparate impact. Calif. L. Rev., Vol. 104 (2016), 671.","journal-title":"Calif. L. Rev."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942287"},{"key":"e_1_3_2_1_6_1","volume-title":"International Conference on Discovery Science. Springer, 52-68","author":"Cantini Riccardo","year":"2024","unstructured":"Riccardo Cantini, Giada Cosenza, Alessio Orsino, and Domenico Talia. 2024. Are Large Language Models Really Bias-Free? Jailbreak Prompts for Assessing Adversarial Robustness to Bias Elicitation. In International Conference on Discovery Science. Springer, 52-68."},{"volume-title":"Statistical inference","author":"Casella George","key":"e_1_3_2_1_7_1","unstructured":"George Casella and Roger Berger. 2024. Statistical inference. CRC press."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/0093854819874090"},{"key":"e_1_3_2_1_9_1","volume-title":"Sebastian Felix Schwemer, and Anders S\u00f8gaard","author":"Chalkidis Ilias","year":"2022","unstructured":"Ilias Chalkidis, Tommaso Pasini, Sheng Zhang, Letizia Tomada, Sebastian Felix Schwemer, and Anders S\u00f8gaard. 2022. Fairlex: A multilingual benchmark for evaluating fairness in legal text processing. arXiv preprint arXiv:2203.07228 (2022)."},{"key":"e_1_3_2_1_10_1","first-page":"1249","article-title":"Technological due process","volume":"85","author":"Citron Danielle Keats","year":"2007","unstructured":"Danielle Keats Citron. 2007. Technological due process. Wash. UL Rev., Vol. 85 (2007), 1249.","journal-title":"Wash. UL Rev."},{"key":"e_1_3_2_1_11_1","volume-title":"2016 Stata Conference. Stata Users Group.","author":"Correia Sergio","year":"2016","unstructured":"Sergio Correia. 2016. reghdfe: Estimating linear models with multi-way fixed effects. In 2016 Stata Conference. Stata Users Group."},{"key":"e_1_3_2_1_12_1","volume-title":"Chatlaw: Open-source legal large language model with integrated external knowledge bases. CoRR","author":"Cui Jiaxi","year":"2023","unstructured":"Jiaxi Cui, Zongjian Li, Yang Yan, Bohua Chen, and Li Yuan. 2023. Chatlaw: Open-source legal large language model with integrated external knowledge bases. CoRR (2023)."},{"key":"e_1_3_2_1_13_1","first-page":"1","article-title":"A view of how language models will transform law","volume":"92","author":"Fagan Frank","year":"2024","unstructured":"Frank Fagan. 2024. A view of how language models will transform law. Tenn. L. Rev., Vol. 92 (2024), 1.","journal-title":"Tenn. L. Rev."},{"key":"e_1_3_2_1_14_1","volume-title":"Chatglm: A family of large language models from glm-130b to glm-4 all tools. arXiv preprint arXiv:2406.12793","author":"Aohan Zeng Team GLM","year":"2024","unstructured":"Team GLM, Aohan Zeng, Bin Xu, Bowen Wang, Chenhui Zhang, Da Yin, Dan Zhang, Diego Rojas, Guanyu Feng, Hanlin Zhao, et al., 2024. Chatglm: A family of large language models from glm-130b to glm-4 all tools. arXiv preprint arXiv:2406.12793 (2024)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-023-09374-7"},{"key":"e_1_3_2_1_16_1","volume-title":"Trial: Evaluating Judicial Fairness for Large Language Models. arXiv preprint arXiv:2507.10852","author":"Hu Yiran","year":"2025","unstructured":"Yiran Hu, Zongyue Xue, Haitao Li, Siyuan Zheng, Qingjing Chen, Shaochun Wang, Xihan Zhang, Ning Zheng, Yun Liu, Qingyao Ai, et al., 2025. LLMs on Trial: Evaluating Judicial Fairness for Large Language Models. arXiv preprint arXiv:2507.10852 (2025)."},{"key":"e_1_3_2_1_17_1","volume-title":"Gpt-4o: The cutting-edge advancement in multimodal llm. Authorea Preprints","author":"Islam Raisa","year":"2024","unstructured":"Raisa Islam and Owana Marzia Moushi. 2024. Gpt-4o: The cutting-edge advancement in multimodal llm. Authorea Preprints (2024)."},{"volume-title":"The Ethics Gap in the Engineering of the Future: Moral Challenges for the Technology of Tomorrow","author":"Karthikeyan Rahulrajan","key":"e_1_3_2_1_18_1","unstructured":"Rahulrajan Karthikeyan, Chieh Yi, and Moses Boudourides. 2024. Criminal Justice in the Age of AI: Addressing Bias in Predictive Algorithms Used by Courts. In The Ethics Gap in the Engineering of the Future: Moral Challenges for the Technology of Tomorrow. Emerald Publishing Limited, 27-50."},{"key":"e_1_3_2_1_19_1","volume-title":"Intrinsic self-correction for enhanced morality: An analysis of internal mechanisms and the superficial hypothesis. arXiv preprint arXiv:2407.15286","author":"Liu Guangliang","year":"2024","unstructured":"Guangliang Liu, Haitao Mao, Jiliang Tang, and Kristen Marie Johnson. 2024. Intrinsic self-correction for enhanced morality: An analysis of internal mechanisms and the superficial hypothesis. arXiv preprint arXiv:2407.15286 (2024)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1093\/jla\/laae009"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","unstructured":"Ramya Srinivasan Mehdi Bahrami Ryosuke Sonoda. 2024. LLM Diagnostic Toolkit: Evaluating LLMs for Ethical Issues. (2024). doi: 10.1109\/IJCNN60899.2024.10650995","DOI":"10.1109\/IJCNN60899.2024.10650995"},{"key":"e_1_3_2_1_22_1","volume-title":"Reasoning Beyond Bias: A Study on Counterfactual Prompting and Chain of Thought Reasoning. arXiv preprint arXiv:2408.08651","author":"Moore Kyle","year":"2024","unstructured":"Kyle Moore, Jesse Roberts, Thao Pham, and Douglas Fisher. 2024. Reasoning Beyond Bias: A Study on Counterfactual Prompting and Chain of Thought Reasoning. arXiv preprint arXiv:2408.08651 (2024)."},{"key":"e_1_3_2_1_23_1","volume-title":"United Kingdom Stata Users' Group Meeting. 133-138","author":"Nichols Austin","year":"2007","unstructured":"Austin Nichols and Mark Schaffer. 2007. Clustered errors in Stata. In United Kingdom Stata Users' Group Meeting. 133-138."},{"volume-title":"The black box society: The secret algorithms that control money and information","author":"Pasquale Frank","key":"e_1_3_2_1_24_1","unstructured":"Frank Pasquale. 2015. The black box society: The secret algorithms that control money and information. Harvard University Press."},{"key":"e_1_3_2_1_25_1","volume-title":"Antibody-drug conjugates: current status and future directions. Drug discovery today","author":"Perez Heidi L","year":"2014","unstructured":"Heidi L Perez, Pina M Cardarelli, Shrikant Deshpande, Sanjeev Gangwar, Gretchen M Schroeder, Gregory D Vite, and Robert M Borzilleri. 2014. Antibody-drug conjugates: current status and future directions. Drug discovery today, Vol. 19, 7 (2014), 869-881."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3593979"},{"key":"e_1_3_2_1_27_1","volume-title":"Aequitas: A Bias and Fairness Audit Toolkit. arXiv:1811.05577 [cs.LG] https:\/\/arxiv.org\/abs\/1811.05577","author":"Saleiro Pedro","year":"2019","unstructured":"Pedro Saleiro, Benedict Kuester, Loren Hinkson, Jesse London, Abby Stevens, Ari Anisfeld, Kit T. Rodolfa, and Rayid Ghani. 2019. Aequitas: A Bias and Fairness Audit Toolkit. arXiv:1811.05577 [cs.LG] https:\/\/arxiv.org\/abs\/1811.05577"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3627673.3680020"},{"key":"e_1_3_2_1_29_1","volume-title":"Intersectionality of race, ethnicity, gender, and age on criminal punishment. Sociological perspectives","author":"Steffensmeier Darrell","year":"2017","unstructured":"Darrell Steffensmeier, Noah Painter-Davis, and Jeffery Ulmer. 2017. Intersectionality of race, ethnicity, gender, and age on criminal punishment. Sociological perspectives, Vol. 60, 4 (2017), 810-833."},{"key":"e_1_3_2_1_30_1","volume-title":"Machine learning and law: An overview. Research handbook on big data law","author":"Surden Harry","year":"2021","unstructured":"Harry Surden. 2021. Machine learning and law: An overview. Research handbook on big data law (2021), 171-184."},{"key":"e_1_3_2_1_31_1","volume-title":"Ryan Burnell, Libin Bai, Anmol Gulati, Garrett Tanzer, Damien Vincent, Zhufeng Pan, Shibo Wang, et al.","author":"Team Gemini","year":"2024","unstructured":"Gemini Team, Petko Georgiev, Ving Ian Lei, Ryan Burnell, Libin Bai, Anmol Gulati, Garrett Tanzer, Damien Vincent, Zhufeng Pan, Shibo Wang, et al., 2024. Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context. arXiv preprint arXiv:2403.05530 (2024)."},{"key":"e_1_3_2_1_32_1","unstructured":"Hrishikesh Viswanath and Tianyi Zhang. 2025. FairPy: A Toolkit for Evaluation of Prediction Biases and their Mitigation in Large Language Models. arXiv:2302.05508 [cs.CL] https:\/\/arxiv.org\/abs\/2302.05508"},{"key":"e_1_3_2_1_33_1","volume-title":"Vite - Next Generation Frontend Tooling. https:\/\/github.com\/vitejs\/vite. Accessed","author":"Contributors Vite","year":"2025","unstructured":"Vite Contributors. 2020. Vite - Next Generation Frontend Tooling. https:\/\/github.com\/vitejs\/vite. Accessed: 16 May 2025."},{"key":"e_1_3_2_1_34_1","volume-title":"https:\/\/vuejs.org\/guide\/introduction.html. Accessed","author":"JavaScript The Progressive","year":"2025","unstructured":"Vue.js Developers. 2014. Vue.js - The Progressive JavaScript Framework v3.0. https:\/\/vuejs.org\/guide\/introduction.html. Accessed: 16 May 2025."},{"key":"e_1_3_2_1_35_1","first-page":"131","article-title":"How to argue with an algorithm: Lessons from the COMPAS-ProPublica debate. Colo","volume":"17","author":"Washington Anne L","year":"2018","unstructured":"Anne L Washington. 2018. How to argue with an algorithm: Lessons from the COMPAS-ProPublica debate. Colo. Tech. LJ, Vol. 17 (2018), 131.","journal-title":"Tech. LJ"},{"key":"e_1_3_2_1_36_1","article-title":"Fairlearn: Assessing and Improving Fairness of AI Systems","volume":"24","author":"Weerts Hilde","year":"2023","unstructured":"Hilde Weerts, Miroslav Dud\u00edk, Richard Edgar, Adrin Jalali, Roman Lutz, and Michael Madaio. 2023. Fairlearn: Assessing and Improving Fairness of AI Systems. Journal of Machine Learning Research, Vol. 24 (2023). http:\/\/jmlr.org\/papers\/v24\/23-0389.html","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_37_1","first-page":"1578","volume-title":"Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society","volume":"7","author":"Wilson Kyra","year":"2024","unstructured":"Kyra Wilson and Aylin Caliskan. 2024. Gender, race, and intersectional bias in resume screening via language model retrieval. In Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society, Vol. 7. 1578-1590."},{"key":"e_1_3_2_1_38_1","volume-title":"London Stata Conference","author":"Xu Zhao","year":"2021","unstructured":"Zhao Xu. 2021. PyStata-Python and Stata integration. In London Stata Conference 2021. Stata Users Group."},{"key":"e_1_3_2_1_39_1","first-page":"7527","volume-title":"Proceedings of the Thirty-Third International Joint Conference on Artificial Intelligence, IJCAI-24","author":"Xue Zongyue","year":"2024","unstructured":"Zongyue Xue, Huanghai Liu, Yiran Hu, Yuliang Qian, Yajing Wang, Kangle Kong, Chenlu Wang, Yun Liu, and Weixing Shen. 2024. LEEC for Judicial Fairness: A Legal Element Extraction Dataset with Extensive Extra-Legal Labels. In Proceedings of the Thirty-Third International Joint Conference on Artificial Intelligence, IJCAI-24. 7527-7535."},{"key":"e_1_3_2_1_40_1","unstructured":"An Yang Beichen Zhang Binyuan Hui Bofei Gao Bowen Yu Chengpeng Li Dayiheng Liu Jianhong Tu Jingren Zhou Junyang Lin Keming Lu Mingfeng Xue Runji Lin Tianyu Liu Xingzhang Ren and Zhenru Zhang. 2024. Qwen2.5-Math Technical Report: Toward Mathematical Expert Model via Self-Improvement. arXiv:2409.12122 [cs.CL] https:\/\/arxiv.org\/abs\/2409.12122"}],"event":{"name":"CIKM '25: The 34th ACM International Conference on Information and Knowledge Management","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Seoul Republic of Korea","acronym":"CIKM '25"},"container-title":["Proceedings of the 34th ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3746252.3761491","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T00:24:12Z","timestamp":1765499052000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3746252.3761491"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,10]]},"references-count":40,"alternative-id":["10.1145\/3746252.3761491","10.1145\/3746252"],"URL":"https:\/\/doi.org\/10.1145\/3746252.3761491","relation":{},"subject":[],"published":{"date-parts":[[2025,11,10]]},"assertion":[{"value":"2025-11-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}