{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T16:40:44Z","timestamp":1786120844365,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":105,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,8,8]],"date-time":"2026-08-08T00:00:00Z","timestamp":1786147200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770855.3818647","type":"proceedings-article","created":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T17:15:37Z","timestamp":1786036537000},"page":"13170-13177","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Causal Methods for LLM Development and Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6526-1162","authenticated-orcid":false,"given":"Dennis","family":"Frauen","sequence":"first","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-5183-4375","authenticated-orcid":false,"given":"Marie","family":"Brockschmidt","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8552-6588","authenticated-orcid":false,"given":"Konstantin","family":"Hess","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-9676-5629","authenticated-orcid":false,"given":"Haorui","family":"Ma","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9900-2822","authenticated-orcid":false,"given":"Yuchen","family":"Ma","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-9174-8230","authenticated-orcid":false,"given":"Abdurahman","family":"Maarouf","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-4141-4314","authenticated-orcid":false,"given":"Maresa","family":"Schr\u00f6der","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3725-3821","authenticated-orcid":false,"given":"Jonas","family":"Schweisthal","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0057-6703","authenticated-orcid":false,"given":"Yuxin","family":"Wang","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0021-7667","authenticated-orcid":false,"given":"Athiya","family":"Deviyani","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8400-3732","authenticated-orcid":false,"given":"Sonali","family":"Parbhoo","sequence":"additional","affiliation":[{"name":"Imperial College London, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7955-3956","authenticated-orcid":false,"given":"Rahul G.","family":"Krishnan","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7856-8729","authenticated-orcid":false,"given":"Stefan","family":"Feuerriegel","sequence":"additional","affiliation":[{"name":"MCML, LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,8,8]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"V382, 6671","author":"Angelopoulos Anastasios N.","year":"2023","unstructured":"Anastasios N. Angelopoulos, Stephen Bates, Clara Fannjiang, Michael I. Jordan, and Tijana Zrnic. 2023. Prediction-powered inference. Science, V382, 6671 (2023), 669-674."},{"key":"e_1_3_2_1_2_1","volume-title":"Cost-optimal active AI, model evaluation. VarXiv:2506.07949","author":"Angelopoulos Anastasios N.","year":"2025","unstructured":"Anastasios N. Angelopoulos, Jacob Eisenstein, Jonathan Berant, Alekh Agarwal, and Adam Fisch. 2025. Cost-optimal active AI, model evaluation. VarXiv:2506.07949 (2025)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.3982\/ECTA15732"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Suhana Bedi Hejie Cui Miguel Fuentes Alyssa Unell Michael Wornow Juan M Banda Nikesh Kotecha Timothy Keyes Yifan Mai Mert Oez et al. 2026. Holistic evaluation of large language models for medical tasks with MedHELM. Nature Medicine V32 (2026) 943-951.","DOI":"10.1038\/s41591-025-04151-2"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Emily M. Bender Timnit Gebru Angelina McMillan-Major and Shmargaret Shmitchell. 2021. On the dangers of stochastic parrots: Can language models be too big?. In FAccT.","DOI":"10.1145\/3442188.3445922"},{"key":"e_1_3_2_1_6_1","volume-title":"Wellner","author":"Bickel Peter J.","year":"1998","unstructured":"Peter J. Bickel, Chris A. J. Klaassen, Ya'acov Ritov, and Jon A. Wellner. 1998. Efficient and adaptive estimation for semiparametric models. Springer."},{"key":"e_1_3_2_1_7_1","unstructured":"Matthieu Bou Nyal Patel Arjun Jagota Satyapriya Krishna and Sonali Parbhoo. 2026. The alignment auditor: A Bayesian framework for verifying and refining LLM objectives. In ICLR."},{"key":"e_1_3_2_1_8_1","first-page":"324","article-title":"Rank analysis of incomplete block designs: I. The, method of paired comparisons","volume":"39","author":"Bradley Ralph Allan","year":"1952","unstructured":"Ralph Allan Bradley and Milton E. Terry. 1952. Rank analysis of incomplete block designs: I. The, method of paired comparisons. Biometrika, V39, 3\/4 (1952), 324-345.","journal-title":"Biometrika"},{"key":"e_1_3_2_1_9_1","volume-title":"USENIX Security Symp.","author":"Carlini Nicholas","year":"2021","unstructured":"Nicholas Carlini, Florian Tramer, Eric Wallace, Matthew Jagielski, Ariel Herbert-Voss, Katherine Lee, Adam Roberts, Tom Brown, Dawn Song, Ulfar Erlingsson, et al., 2021. Extracting training data from large language models. In USENIX Security Symp.,"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Ivi Chatzi Eleni Straitouri Suhas Thejaswi and Manuel Gomez Rodriguez. 2024. Prediction-powered ranking of large language models. In NeurIPS.","DOI":"10.52202\/079017-3594"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1111\/ectj.12097"},{"key":"e_1_3_2_1_12_1","volume-title":"Wei-Lin Chiang, Aditya Mittal, Naman Jain, Tianjun Zhang, Ion Stoica, Chris Donahue, and Ameet Talwalkar.","author":"Chi Wayne","year":"2025","unstructured":"Wayne Chi, Valerie Chen, Anastasios Nikolas Angelopoulos, Wei-Lin Chiang, Aditya Mittal, Naman Jain, Tianjun Zhang, Ion Stoica, Chris Donahue, and Ameet Talwalkar. 2025. Copilot Arena: A platform for code LLM evaluation in the wild. In ICML."},{"key":"e_1_3_2_1_13_1","volume-title":"Chatbot Arena: An, open platform for evaluating LLMs, by human preference. In ICML.","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios N Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael I Jordan, Joseph E Gonzalez, and Ion Stoica. 2024. Chatbot Arena: An, open platform for evaluating LLMs, by human preference. In ICML."},{"key":"e_1_3_2_1_14_1","unstructured":"Paul F Christiano Jan Leike Tom Brown Miljan Martic Shane Legg and Dario Amodei. 2017. Deep reinforcement learning from human preferences. In NeurIPS."},{"key":"e_1_3_2_1_15_1","unstructured":"Thomas Cook Alan Mishler and Aaditya Ramdas. 2024. Semiparametric efficient inference in adaptive experiments. In CLeaR."},{"key":"e_1_3_2_1_16_1","unstructured":"Alicia Curth and Mihaela Van der Schaar. 2021. Nonparametric estimation of heterogeneous treatment effects: From theory to learning algorithms. In AISTATS."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.276"},{"key":"e_1_3_2_1_18_1","unstructured":"Dujian Ding Ankur Mallick Chi Wang Robert Sim Subhabrata Mukherjee Victor Ruhle Laks V. S. Lakshmanan and Ahmed Hassan Awadallah. 2024. Hybrid LLM: Cost-efficient and quality-aware query routing. In ICLR."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Lucas Dixon John Li Jeffrey Sorensen Nithum Thain and Lucy Vasserman. 2018. Measuring and mitigating unintended bias in text classification. In AIES.","DOI":"10.1145\/3278721.3278729"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Jesse Dodge Maarten Sap Ana Marasovi\u0107 William Agnew Gabriel Ilharco Dirk Groeneveld Margaret Mitchell and Matt Gardner. 2021. Documenting large webtext corpora: A case study on the colossal clean crawled corpus. In EMNLP.","DOI":"10.18653\/v1\/2021.emnlp-main.98"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2024.2335588"},{"key":"e_1_3_2_1_22_1","unstructured":"Nan Du Yanping Huang Andrew M Dai Simon Tong Dmitry Lepikhin Yuanzhong Xu Maxim Krikun Yanqi Zhou Adams Wei Yu Orhan Firat Barret Zoph Liam Fedus Maarten P Bosma Zongwei Zhou Tao Wang Emma Wang Kellie Webster Marie Pellat Kevin Robinson Kathleen Meier-Hellstern Toju Duke Lucas Dixon Kun Zhang Quoc Le Yonghui Wu Zhifeng Chen and Claire Cui. 2022. GLaM: Efficient scaling of language models with mixture-of-experts. In ICML."},{"key":"e_1_3_2_1_23_1","unstructured":"Miroslav Dud\u00edk John Langford and Lihong Li. 2011. Doubly robust policy evaluation and learning. In ICML."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Amir Feder Katherine A Keith Emaad Manzoor Reid Pryzant Dhanya Sridhar Zach Wood-Doughty Jacob Eisenstein Justin Grimmer Roi Reichart Margaret E Roberts et al. 2022. Causal inference in natural language processing: Estimation prediction interpretation and beyond. In TACL.","DOI":"10.1162\/tacl_a_00511"},{"key":"e_1_3_2_1_25_1","volume-title":"Yuhan Liu, and Yulia Tsvetkov.","author":"Feng Shangbin","year":"2023","unstructured":"Shangbin Feng, Chan Young Park, Yuhan Liu, and Yulia Tsvetkov. 2023. From pretraining data to language models to downstream tasks: Tracking the trails of political biases leading to unfair NLP models. In ACL."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Adam Fisch Joshua Maynez R Alex Hofer Bhuwan Dhingra Amir Globerson and William W Cohen. 2024. Stratified prediction-powered inference for hybrid language model evaluation. In NeurIPS.","DOI":"10.52202\/079017-3541"},{"key":"e_1_3_2_1_27_1","unstructured":"Dennis Frauen Athiya Deviyani Mihaela van der Schaar and Stefan Feuerriegel. 2026 a. Nonparametric LLM evaluation from preference data. In ICML."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Dennis Frauen Valentyn Melnychuk and Stefan Feuerriegel. 2023. Sharp bounds for generalized causal sensitivity analysis. In NeurIPS.","DOI":"10.52202\/075280-1765"},{"key":"e_1_3_2_1_29_1","volume-title":"2026 b. Machine Learning for Causal Inference","author":"Frauen Dennis","unstructured":"Dennis Frauen, Valentyn Melnychuk, Lars van der Laan, and Stefan Feuerriegel. 2026 b. Machine Learning for Causal Inference. In Wiley StatsRef. Wiley."},{"key":"e_1_3_2_1_30_1","volume-title":"Prompt-to-leaderboard: Prompt-adaptive LLM, evaluations. In ICML.","author":"Frick Evan","year":"2025","unstructured":"Evan Frick, Connor Chen, Joseph Tennyson, Tianle Li, Wei-Lin Chiang, Anastasios N. Angelopoulos, and Ion Stoica. 2025. Prompt-to-leaderboard: Prompt-adaptive LLM, evaluations. In ICML."},{"key":"e_1_3_2_1_31_1","unstructured":"Leo Gao Stella Biderman Sid Black Laurence Golding Travis Hoppe Charles Foster Jason Phang Horace He Anish Thite Noa Nabeshima Shawn Presser and Connor Leahy. 2020. The pile: An 800GB dataset of diverse text for language modeling. showeprint2101.00027"},{"key":"e_1_3_2_1_32_1","unstructured":"Yunfan Gao Yun Xiong Xinyu Gao Kangxiang Jia Jinliu Pan Yuxi Bi Yi Dai Jiawei Sun Meng Wang and Haofen Wang. 2024. Retrieval-augmented generation for large language models: A survey. showeprint2312.10997"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-020-00257-z"},{"key":"e_1_3_2_1_34_1","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri et al. 2024. The Llama 3 herd of models. showeprint2407.21783"},{"key":"e_1_3_2_1_35_1","unstructured":"Luke Guerdan Justin Whitehouse Kimberly Truong Ken Holstein and Steven Wu. 2026. Doubly-robust LLM-as-a-judge: Externally valid estimation with imperfect personas. In ICLR."},{"key":"e_1_3_2_1_36_1","unstructured":"Taicheng Guo Xiuying Chen Yaqi Wang Ruidi Chang Shichao Pei Nitesh V Chawla Olaf Wiest and Xiangliang Zhang. 2024. Large language model based multi-agents: A survey of progress and challenges. In IJCAI."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Yuan Guo Peng Tian Jayashree Kalpathy-Cramer Susan Ostmo J.Peter Campbell Michael F.Chiang Deniz Erdogmus Jennifer Dy and Stratis Ioannidis. 2018. Experimental design under the Bradley-Terry model. In IJCAI.","DOI":"10.24963\/ijcai.2018\/304"},{"key":"e_1_3_2_1_38_1","volume-title":"Agentic Markets Workshop at ICML","author":"Hu Qitian Jason","year":"2024","unstructured":"Qitian Jason Hu, Jacob Bieker, Xiuyu Li, Nan Jiang, Benjamin Keigwin, Gaurav Ranganath, Kurt Keutzer, and Shriyash Kaustubh Upadhyay. 2024. RouterBench: A benchmark for multi-LLM routing system. In Agentic Markets Workshop at ICML 2024."},{"key":"e_1_3_2_1_39_1","volume-title":"Probellm: Automating Principled Diagnosis of LLM Failures. In ICML.","author":"Huang Yue","year":"2026","unstructured":"Yue Huang, Zhengzhe Jiang, Yuchen Ma, Yu Jiang, Xiangqi Wang, Yujun Zhou, Yuexing Hao, Kehan Guo, Pin-Yu Chen, Stefan Feuerriegel, et al., 2026. Probellm: Automating Principled Diagnosis of LLM Failures. In ICML."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Gautier Izacard and Edouard Grave. 2021. Leveraging passage retrieval with generative models for open domain question answering. In EACL.","DOI":"10.18653\/v1\/2021.eacl-main.74"},{"key":"e_1_3_2_1_41_1","unstructured":"Kevin G Jamieson and Robert Nowak. 2011. Active ranking using pairwise comparisons. In NeurIPS."},{"key":"e_1_3_2_1_42_1","volume-title":"Max Kleiman-Weiner, Mrinmaya Sachan, et al.","author":"Jin Zhijing","year":"2023","unstructured":"Zhijing Jin, Yuen Chen, Felix Leeb, Luigi Gresele, Ojasv Kamal, LYU Zhiheng, Kevin Blin, Fernando Gonzalez Adauto, Max Kleiman-Weiner, Mrinmaya Sachan, et al., 2023. CLadder: Assessing causal reasoning in language models. In NeurIPS."},{"key":"e_1_3_2_1_43_1","unstructured":"Zhijing Jin Jiarui Liu Zhiheng Lyu Spencer Poff Mrinmaya Sachan Rada Mihalcea Mona Diab and Bernhard Sch\u00f6lkopf. 2024. Can large language models infer causation from correlation?. In ICLR."},{"key":"e_1_3_2_1_44_1","volume-title":"Second Conf. on Language Modeling.","author":"Joselowitz Jared","year":"2025","unstructured":"Jared Joselowitz, Ritam Majumdar, Arjun Jagota, Matthieu Bou, Nyal Patel, Satyapriya Krishna, and Sonali Parbhoo. 2025. Insights from the inverse: reconstructing LLM training goals through inverse reinforcement learning. In Second Conf. on Language Modeling."},{"key":"e_1_3_2_1_45_1","volume-title":"V9","author":"Kaddour Jean","year":"2025","unstructured":"Jean Kaddour, Aengus Lynch, Qi Liu, Matt J. Kusner, and Ricardo Silva. 2025. Causal Machine learning: A survey and open problems. Foundations and Trends in Optimization, V9 (2025), 1-247. Issue 1-2."},{"key":"e_1_3_2_1_46_1","unstructured":"Jean Kaddour Yuchen Zhu Qi Liu Matt J Kusner and Ricardo Silva. 2021. Causal effect inference for structured treatments. In NeurIPS."},{"key":"e_1_3_2_1_47_1","volume-title":"VarXiv:2512.21917","author":"Kallus Nathan","year":"2025","unstructured":"Nathan Kallus. 2025. Semiparametric preference optimization: Your, language model is secretly a single-index model. arXiv preprint, VarXiv:2512.21917 (2025)."},{"key":"e_1_3_2_1_48_1","unstructured":"Nikhil Kandpal Eric Wallace and Colin Raffel. 2022. Deduplicating training data mitigates privacy risks in language models. In ICML."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"Vladimir Karpukhin Barlas Oguz Sewon Min Patrick Lewis Ledell Wu Sergey Edunov Danqi Chen and Wen-tau Yih. 2020. Dense passage retrieval for open-domain question answering. In EMNLP.","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"e_1_3_2_1_50_1","unstructured":"Chinmaya Kausik Yangyi Lu Kevin Tan Maggie Makar Yixin Wang and Ambuj Tewari. 2024. Offline policy evaluation and optimization under confounding. In AISTATS."},{"key":"e_1_3_2_1_51_1","volume-title":"Semiparametric doubly robust targeted double machine learning: a review. 2203.06469","author":"Kennedy Edward H.","year":"2023","unstructured":"Edward H. Kennedy. 2023a. Semiparametric doubly robust targeted double machine learning: a review. 2203.06469 (2023)."},{"key":"e_1_3_2_1_52_1","volume-title":"V17, 2","author":"Kennedy Edward H.","year":"2023","unstructured":"Edward H. Kennedy. 2023b. Towards optimal doubly robust estimation of heterogeneous causal effects. Electronic Journal of Statistics, V17, 2 (2023), 3008-3049."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1111\/rssb.12212"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1038\/s43588-025-00814-9"},{"key":"e_1_3_2_1_55_1","volume-title":"Causal reasoning and large language models: Opening a new frontier for causality. TMLR","author":"Kiciman Emre","year":"2023","unstructured":"Emre Kiciman, Robert Ness, Amit Sharma, and Chenhao Tan. 2023. Causal reasoning and large language models: Opening a new frontier for causality. TMLR (2023)."},{"key":"e_1_3_2_1_56_1","unstructured":"Peter Kirgis Sayash Kapoor Stephan Rabanser Nitya Nadgir Cozmin Ududec Magda Dubois JJ Allaire Conrad Stosz Marius Hobbhahn Jacob Steinhardt et al. 2026. Log analysis is necessary for credible evaluation of AI agents. arXiv:2605.08545 (2026)."},{"key":"e_1_3_2_1_57_1","volume-title":"Yuta Saito, and Thorsten Joachims.","author":"Kiyohara Haruka","year":"2025","unstructured":"Haruka Kiyohara, Daniel Yiming Cao, Yuta Saito, and Thorsten Joachims. 2025. An Off-Policy Learning Approach for Steering Sentence Generation towards Personalization. In RecSys."},{"key":"e_1_3_2_1_58_1","unstructured":"Kasia Kobalczyk and Mihaela van der Schaar. 2025. Preference learning for AI alignment: A causal perspective. In ICML."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.29"},{"key":"e_1_3_2_1_60_1","volume-title":"Causal inference with high-dimensional treatments. arXiv:2602.21423","author":"Kramer Patrick","year":"2026","unstructured":"Patrick Kramer, Edward H Kennedy, and Isaac M Opper. 2026. Causal inference with high-dimensional treatments. arXiv:2602.21423 (2026)."},{"key":"e_1_3_2_1_61_1","unstructured":"Katherine Lee Daphne Ippolito Andrew Nystrom Chiyuan Zhang Douglas Eck Chris Callison-Burch and Nicholas Carlini. 2022. Deduplicating training data makes language models better. In ACL."},{"key":"e_1_3_2_1_62_1","volume-title":"Stephen Anthony Rose, and Jesse C. Cresswell","author":"Leung Kin Kwan","year":"2026","unstructured":"Kin Kwan Leung, Mouloud Belbahri, Yi Sui, Alex Labach, Xueying Zhang, Stephen Anthony Rose, and Jesse C. Cresswell. 2026. Classifying and addressing the diversity of errors in retrieval-augmented generation systems. In EACL."},{"key":"e_1_3_2_1_63_1","unstructured":"Patrick Lewis Ethan Perez Aleksandra Piktus Fabio Petroni Vladimir Karpukhin Naman Goyal Heinrich K\u00fcttler Mike Lewis Wen-tau Yih Tim Rockt\u00e4schel Sebastian Riedel and Douwe Kiela. 2020. Retrieval-augmented generation for knowledge-intensive NLP tasks. In NeurIPS."},{"key":"e_1_3_2_1_64_1","volume-title":"Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. arXiv:2503.18256","author":"Li Xiudi","year":"2025","unstructured":"Xiudi Li and Sijia Li. 2025. Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. arXiv:2503.18256 (2025)."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.417"},{"key":"e_1_3_2_1_66_1","volume-title":"Gonzalez","author":"Miroyan Mihran","year":"2025","unstructured":"Mihran Miroyan, Tsung-Han Wu, Logan King, Tianle Li, Jiayi Pan, Xinyan Hu, Wei-Lin Chiang, Anastasios N. Angelopoulos, Trevor Darrell, Narges Norouzi, and Joseph E. Gonzalez. 2025. Search Arena: analyzing search-augmented LLMs. arXiv:2506.05334 (2025)."},{"key":"e_1_3_2_1_67_1","volume-title":"B-learner: Quasi-oracle bounds on heterogeneous causal effects under hidden confounding. In ICML.","author":"Oprescu Miruna","year":"2023","unstructured":"Miruna Oprescu, Jacob Dorn, Marah Ghoummaid, Andrew Jesson, Nathan Kallus, and Uri Shalit. 2023. B-learner: Quasi-oracle bounds on heterogeneous causal effects under hidden confounding. In ICML."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"crossref","unstructured":"Long Ouyang Jeff Wu Xu Jiang Diogo Almeida Carroll L Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell Peter Welinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. In NeurIPS.","DOI":"10.52202\/068431-2011"},{"key":"e_1_3_2_1_69_1","unstructured":"Charles Packer Vivian Fang Shishir_G Patil Kevin Lin Sarah Wooders and Joseph_E Gonzalez. 2023. MemGPT: towards LLMs as operating systems. (2023)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.57"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606763"},{"key":"e_1_3_2_1_72_1","unstructured":"Judea Pearl. 2009. Causality. Cambridge Univ. Press NYC."},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"crossref","unstructured":"Rafael Rafailov Archit Sharma Eric Mitchell Stefano Ermon Christopher D Manning and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. In NeurIPS.","DOI":"10.52202\/075280-2338"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1994.10476818"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0037350"},{"key":"e_1_3_2_1_76_1","unstructured":"Yuta Saito and Thorsten Joachims. 2022. Off-policy evaluation for large action spaces via embeddings. In ICML."},{"key":"e_1_3_2_1_77_1","unstructured":"Yuta Saito Qingyang Ren and Thorsten Joachims. 2023. Off-policy evaluation for large action spaces via conjunct effect modeling. In ICML."},{"key":"e_1_3_2_1_78_1","volume-title":"POTEC: Off-policy contextual bandits for large action spaces via policy decomposition. In ICLR.","author":"Saito Yuta","year":"2025","unstructured":"Yuta Saito, Jihan Yao, and Thorsten Joachims. 2025. POTEC: Off-policy contextual bandits for large action spaces via policy decomposition. In ICLR."},{"key":"e_1_3_2_1_79_1","volume-title":"Smith","author":"Sap Maarten","year":"2019","unstructured":"Maarten Sap, Dallas Card, Saadia Gabriel, Yejin Choi, and Noah A. Smith. 2019. The risk of racial bias in hate speech detection. In ACL."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"crossref","unstructured":"Ayush Sawarni Sahasrajit Sarmasarkar and Vasilis Syrgkanis. 2025. Preference learning with response time: Robust losses and guarantees. In NeurIPS.","DOI":"10.52202\/085713-3477"},{"key":"e_1_3_2_1_81_1","volume-title":"Toolformer: Language Models can teach themselves to use tools. In NeurIPS.","author":"Schick Timo","year":"2023","unstructured":"Timo Schick, Jane Dwivedi-Yu, Roberto Dessi, Roberta Raileanu, Maria Lomeli, Eric Hambro, Luke Zettlemoyer, Nicola Cancedda, and Thomas Scialom. 2023. Toolformer: Language Models can teach themselves to use tools. In NeurIPS."},{"key":"e_1_3_2_1_82_1","volume-title":"OrthoBO: Orthogonal Bayesian hyperparameter optimization. arXiv:2605.06454","author":"Schr\u00f6der Maresa","year":"2026","unstructured":"Maresa Schr\u00f6der, Pascal Janetzky, Michael Klar, and Stefan Feuerriegel. 2026. OrthoBO: Orthogonal Bayesian hyperparameter optimization. arXiv:2605.06454 (2026)."},{"key":"e_1_3_2_1_83_1","unstructured":"Uri Shalit Fredrik D Johansson and David Sontag. 2017. Estimating individual treatment effect: Generalization bounds and algorithms. In ICML."},{"key":"e_1_3_2_1_84_1","volume-title":"Reflexion: Language agents with verbal reinforcement learning. In NeurIPS.","author":"Shinn Noah","year":"2023","unstructured":"Noah Shinn, Federico Cassano, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao. 2023. Reflexion: Language agents with verbal reinforcement learning. In NeurIPS."},{"key":"e_1_3_2_1_85_1","volume-title":"Semiparametric plug-in estimation, sup-norm risk bounds, marginal optimization, and inference in BTL, model. arXiv:2503.15045","author":"Spokoiny Vladimir","year":"2025","unstructured":"Vladimir Spokoiny. 2025. Semiparametric plug-in estimation, sup-norm risk bounds, marginal optimization, and inference in BTL, model. arXiv:2503.15045 (2025)."},{"key":"e_1_3_2_1_86_1","volume-title":"Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth\u00e9e Lacroix, Baptiste Rozi\u00e8re, Naman Goyal, Eric Hambro, Faisal Azhar, et al., 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"crossref","unstructured":"Asterios Tsiourvas Wei Sun and Georgia Perakis. 2025. Causal LLM routing: End-to-end regret minimization from observational data. In NeurIPS.","DOI":"10.52202\/085713-1252"},{"key":"e_1_3_2_1_88_1","volume-title":"NeurIPS, 2025 Workshop: Reliable ML, from Unreliable Data.","author":"Annie","unstructured":"Annie S. Ulichney and Amanda Lee Coston. 2025. Double machine learning evaluation under distribution shift and selection bias. In NeurIPS, 2025 Workshop: Reliable ML, from Unreliable Data."},{"key":"e_1_3_2_1_89_1","volume-title":"Calibeating prediction-powered inference. arXiv:2604.21260","author":"van der Laan Lars","year":"2026","unstructured":"Lars van der Laan and Mark Van Der Laan. 2026. Calibeating prediction-powered inference. arXiv:2604.21260 (2026)."},{"key":"e_1_3_2_1_90_1","volume-title":"Rubin","author":"van der Laan Mark J.","year":"2006","unstructured":"Mark J. van der Laan, and Donald B. Rubin. 2006. Targeted maximum likelihood learning. The International Journal of Biostatistics, V2, 1 (2006)."},{"key":"e_1_3_2_1_91_1","volume-title":"Asymptotic statistics","author":"van der Vaart Aart","unstructured":"Aart van der Vaart. 1998. Asymptotic statistics. Cambridge Univ. Press, Cambridge."},{"key":"e_1_3_2_1_92_1","unstructured":"Victor Veitch Dhanya Sridhar and David Blei. 2020. Adapting text embeddings for causal inference. In UAI."},{"key":"e_1_3_2_1_93_1","volume-title":"Voyager: An open-ended embodied agent with large language models. TMLR","author":"Wang Guanzhi","year":"2024","unstructured":"Guanzhi Wang, Yuqi Xie, Yunfan Jiang, Ajay Mandlekar, Chaowei Xiao, Yuke Zhu, Linxi Fan, and Anima Anandkumar. 2024b. Voyager: An open-ended embodied agent with large language models. TMLR (2024)."},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"crossref","unstructured":"Peiyi Wang Lei Li Liang Chen Zefan Cai Dawei Zhu Binghuai Lin Yunbo Cao Lingpeng Kong Qi Liu Tianyu Liu and Zhifang Sui. 2024a. Large language models are not fair evaluators. In ACL.","DOI":"10.18653\/v1\/2024.acl-long.511"},{"key":"e_1_3_2_1_95_1","volume-title":"Nurendra Choudhary, Edward W Huang, Karthik Subbian, Linjun Zhang, Diyi Yang, James Zou, and Jure Leskovec.","author":"Wu Shirley","year":"2026","unstructured":"Shirley Wu, Parth Sarthi, Shiyu Zhao, Aaron Lee, Herumb Shandilya, Adrian Mladenic Grobelnik, Nurendra Choudhary, Edward W Huang, Karthik Subbian, Linjun Zhang, Diyi Yang, James Zou, and Jure Leskovec. 2026. Optimas: Optimizing compound AI systems with globally aligned local rewards. In ICLR."},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"crossref","unstructured":"Yu Xia Tong Yu Zhankui He Handong Zhao Julian McAuley and Shuai Li. 2024. Aligning as debiasing: Causality-aware Alignment via reinforcement learning with interventional feedback. In NAACL.","DOI":"10.18653\/v1\/2024.naacl-long.262"},{"key":"e_1_3_2_1_97_1","unstructured":"Erhan Xu Kai Ye Hongyi Zhou Luhan Zhu Francesco Quinzan and Chengchun Shi. 2025b. Doubly robust alignment for large language models. In NeurIPS."},{"key":"e_1_3_2_1_98_1","unstructured":"Yi Xu Laura Ruis Tim Rockt\u00e4schel and Robert Kirk. 2025a. Investigating non-transitivity in LLM-as-a-judge. In ICML."},{"key":"e_1_3_2_1_99_1","unstructured":"Shunyu Yao Jeffrey Zhao Dian Yu Nan Du Izhak Shafran Karthik R Narasimhan and Yuan Cao. 2023. ReAct: Synergizing reasoning and acting in language models. In ICLR."},{"key":"e_1_3_2_1_100_1","volume-title":"Devendra Singh Dhami, and Kristian Kersting","author":"Ze\u010devi\u0107 Matej","year":"2023","unstructured":"Matej Ze\u010devi\u0107, Moritz Willig, Devendra Singh Dhami, and Kristian Kersting. 2023. Causal parrots: Large language models may talk causality but are not causal. TMLR (2023)."},{"key":"e_1_3_2_1_101_1","unstructured":"Guanhua Zhang and Moritz Hardt. 2024. Inherent trade-offs between diversity and stability in multi-task benchmarks. In ICML."},{"key":"e_1_3_2_1_102_1","unstructured":"Yichi Zhang Fangzheng Xie Shu Yang and Chong Wu. 2026. Meta-Router: Bridging gold-standard and preference-based evaluations in LLM routing. In ICLR."},{"key":"e_1_3_2_1_103_1","unstructured":"Wayne Xin Zhao Kun Zhou Junyi Li Tianyi Tang Xiaolei Wang Yupeng Hou Yingqian Min Beichen Zhang Junjie Zhang Zican Dong et al. 2023. A survey of large language models. arXiv:2303.18223 (2023)."},{"key":"e_1_3_2_1_104_1","unstructured":"Shuyan Zhou Frank F. Xu Hao Zhu Xuhui Zhou Robert Lo Abishek Sridhar Xianyi Cheng Tianyue Ou Yonatan Bisk Daniel Fried Uri Alon and Graham Neubig. 2024. WebArena: A realistic web environment for building autonomous agents. In ICLR."},{"key":"e_1_3_2_1_105_1","volume-title":"Cand\u00e8s","author":"Zrnic Tijana","year":"2024","unstructured":"Tijana Zrnic and Emmanuel J. Cand\u00e8s. 2024. Cross-prediction-powered inference. Proc. of the National Academy of Sciences, V121, 15 (2024), e2322083121."}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"deposited":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T01:30:32Z","timestamp":1786066232000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770855.3818647"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,8]]},"references-count":105,"alternative-id":["10.1145\/3770855.3818647","10.1145\/3770855"],"URL":"https:\/\/doi.org\/10.1145\/3770855.3818647","relation":{},"subject":[],"published":{"date-parts":[[2026,8,8]]},"assertion":[{"value":"2026-08-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}