{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,24]],"date-time":"2026-04-24T05:36:12Z","timestamp":1777008972615,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","funder":[{"name":"The Clarendon Fund"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3715275.3732068","type":"proceedings-article","created":{"date-parts":[[2025,6,23]],"date-time":"2025-06-23T17:03:13Z","timestamp":1750698193000},"page":"1048-1059","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Reward Model Interpretability via Optimal and Pessimal Tokens"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5277-8939","authenticated-orcid":false,"given":"Brian","family":"Christian","sequence":"first","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7419-5993","authenticated-orcid":false,"given":"Hannah Rose","family":"Kirk","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0468-5097","authenticated-orcid":false,"given":"Jessica A.F.","family":"Thompson","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2941-2653","authenticated-orcid":false,"given":"Christopher","family":"Summerfield","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5761-8536","authenticated-orcid":false,"given":"Tsvetomira","family":"Dumbalska","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,6,23]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan Nicholas Joseph Saurav Kadavath Jackson Kernion Tom Conerly Sheer El-Showk Nelson Elhage Zac Hatfield-Dodds Danny Hernandez Tristan Hume Scott Johnston Shauna Kravec Liane Lovitt Neel Nanda Catherine Olsson Dario Amodei Tom Brown Jack Clark Sam McCandlish Chris Olah Ben Mann and Jared Kaplan. 2022. Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback. arxiv:https:\/\/arXiv.org\/abs\/2204.05862\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2204.05862"},{"key":"e_1_3_3_3_3_2","unstructured":"Meriem Boubdir Edward Kim Beyza Ermis Sara Hooker and Marzieh Fadaee. 2023. Elo Uncovered: Robustness and Best Practices in Language Model Evaluation. arxiv:https:\/\/arXiv.org\/abs\/2311.17295\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2311.17295"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"crossref","unstructured":"Ralph\u00a0Allan Bradley and Milton\u00a0E Terry. 1952. Rank analysis of incomplete block designs: I. The method of paired comparisons. Biometrika 39 3\/4 (1952) 324\u2013345.","DOI":"10.1093\/biomet\/39.3-4.324"},{"key":"e_1_3_3_3_5_2","unstructured":"Stephen Casper Xander Davies Claudia Shi Thomas\u00a0Krendl Gilbert J\u00e9r\u00e9my Scheurer Javier Rando Rachel Freedman Tomasz Korbak David Lindner Pedro Freire et\u00a0al. 2023. Open problems and fundamental limitations of reinforcement learning from human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.15217 (2023)."},{"key":"e_1_3_3_3_6_2","unstructured":"Wei-Lin Chiang Lianmin Zheng Ying Sheng Anastasios\u00a0Nikolas Angelopoulos Tianle Li Dacheng Li Hao Zhang Banghua Zhu Michael Jordan Joseph\u00a0E. Gonzalez and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. arxiv:https:\/\/arXiv.org\/abs\/2403.04132\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2403.04132"},{"key":"e_1_3_3_3_7_2","volume-title":"Advances in Neural Information Processing Systems","author":"Christiano Paul\u00a0F","year":"2017","unstructured":"Paul\u00a0F Christiano, Jan Leike, Tom Brown, Miljan Martic, Shane Legg, and Dario Amodei. 2017. Deep Reinforcement Learning from Human Preferences. In Advances in Neural Information Processing Systems, I.\u00a0Guyon, U.\u00a0Von Luxburg, S.\u00a0Bengio, H.\u00a0Wallach, R.\u00a0Fergus, S.\u00a0Vishwanathan, and R.\u00a0Garnett (Eds.), Vol.\u00a030."},{"key":"e_1_3_3_3_8_2","volume-title":"There are Only Four Billion Floats\u2013So Test Them All!","author":"Dawson Bruce","year":"2014","unstructured":"Bruce Dawson. 2014. There are Only Four Billion Floats\u2013So Test Them All!https:\/\/randomascii.wordpress.com\/2014\/01\/27\/theres-only-four-billion-floatsso-test-them-all\/"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3278721.3278729"},{"key":"e_1_3_3_3_10_2","unstructured":"Nicolai Dorka. 2024. Quantile Regression for Distributional Reward Models in RLHF. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.10164 (2024)."},{"key":"e_1_3_3_3_11_2","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan et\u00a0al. 2024. The llama 3 herd of models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.21783 (2024)."},{"key":"e_1_3_3_3_12_2","unstructured":"Yann Dubois Chen\u00a0Xuechen Li Rohan Taori Tianyi Zhang Ishaan Gulrajani Jimmy Ba Carlos Guestrin Percy\u00a0S Liang and Tatsunori\u00a0B Hashimoto. 2023. AlpacaFarm: A simulation framework for methods that learn from human feedback. Advances in Neural Information Processing Systems 36 (2023) 30039\u201330069."},{"key":"e_1_3_3_3_13_2","unstructured":"Arpad\u00a0E. Elo. 1967. The Proposed USCF Rating System Its Development Theory and Applications. Chess Life 22 8 (August 1967) 242\u2013247."},{"key":"e_1_3_3_3_14_2","first-page":"10835","volume-title":"International Conference on Machine Learning","author":"Gao Leo","year":"2023","unstructured":"Leo Gao, John Schulman, and Jacob Hilton. 2023. Scaling laws for reward model overoptimization. In International Conference on Machine Learning. PMLR, 10835\u201310866."},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604698"},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"crossref","unstructured":"Geoffrey\u00a0E Hinton. 2002. Training products of experts by minimizing contrastive divergence. Neural Computation 14 8 (2002) 1771\u20131800.","DOI":"10.1162\/089976602760128018"},{"key":"e_1_3_3_3_17_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Hosking Tom","year":"2024","unstructured":"Tom Hosking, Phil Blunsom, and Max Bartolo. 2024. Human Feedback is not Gold Standard. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_3_3_18_2","unstructured":"Hamish Ivison Yizhong Wang Jiacheng Liu Zeqiu Wu Valentina Pyatkin Nathan Lambert Noah\u00a0A Smith Yejin Choi and Hanna Hajishirzi. 2024. Unpacking DPO and PPO: Disentangling best practices for learning from preference feedback. Advances in Neural Information Processing Systems 37 (2024) 36602\u201336633."},{"key":"e_1_3_3_3_19_2","unstructured":"Natasha Jaques Asma Ghandeharioun Judy\u00a0Hanwen Shen Craig Ferguson Agata Lapedriza Noah Jones Shixiang Gu and Rosalind Picard. 2019. Way off-policy batch deep reinforcement learning of implicit human preferences in dialog. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1907.00456 (2019)."},{"key":"e_1_3_3_3_20_2","unstructured":"Albert\u00a0Q Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra\u00a0Singh Chaplot Diego de\u00a0las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier et\u00a0al. 2023. Mistral 7B. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.06825 (2023)."},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.148"},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.35"},{"key":"e_1_3_3_3_23_2","unstructured":"Hannah\u00a0Rose Kirk Bertie Vidgen Paul R\u00f6ttger and Scott\u00a0A Hale. 2023. The empty signifier problem: Towards clearer paradigms for operationalising\" alignment\" in large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.02457 (2023)."},{"key":"e_1_3_3_3_24_2","volume-title":"The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track","author":"Kirk Hannah\u00a0Rose","year":"2024","unstructured":"Hannah\u00a0Rose Kirk, Alexander Whitefield, Paul R\u00f6ttger, Andrew\u00a0Michael Bean, Katerina Margatina, Rafael Mosquera, Juan\u00a0Manuel Ciro, Max Bartolo, Adina Williams, He He, Bertie Vidgen, and Scott\u00a0A. Hale. 2024. The PRISM Alignment Dataset: What Participatory, Representative and Individualised Human Feedback Reveals About the Subjective and Multicultural Alignment of Large Language Models. In The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track. https:\/\/openreview.net\/forum?id=DFr5hteojx"},{"key":"e_1_3_3_3_25_2","volume-title":"The Dictionary of Obscure Sorrows","author":"Koenig John","year":"2021","unstructured":"John Koenig. 2021. The Dictionary of Obscure Sorrows. Simon and Schuster."},{"key":"e_1_3_3_3_26_2","unstructured":"Nathan Lambert Thomas\u00a0Krendl Gilbert and Tom Zick. 2023. Entangled preferences: The history and risks of reinforcement learning and human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.13595 (2023)."},{"key":"e_1_3_3_3_27_2","unstructured":"Nathan Lambert Valentina Pyatkin Jacob Morrison LJ Miranda Bill\u00a0Yuchen Lin Khyathi Chandu Nouha Dziri Sachin Kumar Tom Zick Yejin Choi et\u00a0al. 2024. RewardBench: Evaluating reward models for language modeling. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.13787 (2024)."},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"publisher","DOI":"10.4324\/9781315840161"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","unstructured":"Xiao Li Zhuhong Li Qiongxiu Li Bingze Lee Jinghao Cui and Xiaolin Hu. 2024. Faster-GCG: Efficient Discrete Optimization Jailbreak Attacks against Aligned Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.15362 (Oct. 2024). https:\/\/doi.org\/10.48550\/arXiv.2410.15362 arXiv:https:\/\/arXiv.org\/abs\/2410.15362 [cs].","DOI":"10.48550\/arXiv.2410.15362"},{"key":"e_1_3_3_3_30_2","unstructured":"Xinyu Li Ruiyang Zhou Zachary\u00a0C Lipton and Liu Leqi. 2024. Personalized language modeling from personalized human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.05133 (2024)."},{"key":"e_1_3_3_3_31_2","volume-title":"Sentiment analysis and opinion mining","author":"Liu Bing","year":"2022","unstructured":"Bing Liu. 2022. Sentiment analysis and opinion mining. Springer Nature."},{"key":"e_1_3_3_3_32_2","unstructured":"Chris\u00a0Yuhao Liu Liang Zeng Jiacai Liu Rui Yan Jujie He Chaojie Wang Shuicheng Yan Yang Liu and Yahui Zhou. 2024. Skywork-Reward: Bag of Tricks for Reward Modeling in LLMs. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.18451 (2024)."},{"key":"e_1_3_3_3_33_2","unstructured":"Xingzhou Lou Dong Yan Wei Shen Yuzi Yan Jian Xie and Junge Zhang. 2024. Uncertainty-aware Reward Model: Teaching Reward Models to Know What is Unknown. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.00847 (2024)."},{"key":"e_1_3_3_3_34_2","unstructured":"Luke Marks Amir Abdullah Clement Neo Rauno Arike David Krueger Philip Torr and Fazl Barez. 2024. Interpreting Learned Feedback Patterns in Large Language Models. Advances in Neural Information Processing Systems 37 (2024) 36541\u201336566."},{"key":"e_1_3_3_3_35_2","unstructured":"Vivek Myers Evan Ellis Sergey Levine Benjamin Eysenbach and Anca Dragan. 2024. Learning to assist humans without inferring rewards. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.02623 (2024)."},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"e_1_3_3_3_37_2","unstructured":"F.\u00a0\u00c5. Nielsen. 2011. AFINN. http:\/\/www2.compute.dtu.dk\/pubdb\/pubs\/6010-full.html"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"crossref","unstructured":"Chris Olah Alexander Mordvintsev and Ludwig Schubert. 2017. Feature visualization. Distill 2 11 (2017) e7.","DOI":"10.23915\/distill.00007"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1302"},{"key":"e_1_3_3_3_40_2","unstructured":"Sriyash Poddar Yanming Wan Hamish Ivison Abhishek Gupta and Natasha Jaques. 2024. Personalizing reinforcement learning from human feedback with variational preference learning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2408.10075 (2024)."},{"key":"e_1_3_3_3_41_2","unstructured":"Rafael Rafailov Archit Sharma Eric Mitchell Christopher\u00a0D Manning Stefano Ermon and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. Advances in Neural Information Processing Systems 36 (2023) 53728\u201353741."},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1163"},{"key":"e_1_3_3_3_43_2","unstructured":"John Schulman Filip Wolski Prafulla Dhariwal Alec Radford and Oleg Klimov. 2017. Proximal policy optimization algorithms. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1707.06347 (2017)."},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"crossref","unstructured":"Eldar Shafir Itamar Simonson and Amos Tversky. 1993. Reason-based choice. Cognition 49 1-2 (1993) 11\u201336.","DOI":"10.1016\/0010-0277(93)90034-S"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"crossref","unstructured":"Wei Shen Rui Zheng Wenyu Zhan Jun Zhao Shihan Dou Tao Gui Qi Zhang and Xuanjing Huang. 2023. Loose lips sink ships: Mitigating length bias in reinforcement learning from human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.05199 (2023).","DOI":"10.18653\/v1\/2023.findings-emnlp.188"},{"key":"e_1_3_3_3_46_2","unstructured":"Prasann Singhal Tanya Goyal Jiacheng Xu and Greg Durrett. 2023. A long way to go: Investigating length correlations in RLHF. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.03716 (2023)."},{"key":"e_1_3_3_3_47_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Siththaranjan Anand","year":"2024","unstructured":"Anand Siththaranjan, Cassidy Laidlaw, and Dylan Hadfield-Menell. 2024. Distributional Preference Learning: Understanding and Accounting for Hidden Context in RLHF. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_3_3_48_2","unstructured":"Taylor Sorensen Jared Moore Jillian Fisher Mitchell Gordon Niloofar Mireshghallah Christopher\u00a0Michael Rytting Andre Ye Liwei Jiang Ximing Lu Nouha Dziri et\u00a0al. 2024. A roadmap to pluralistic alignment. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.05070 (2024)."},{"key":"e_1_3_3_3_49_2","unstructured":"Nisan Stiennon Long Ouyang Jeffrey Wu Daniel Ziegler Ryan Lowe Chelsea Voss Alec Radford Dario Amodei and Paul\u00a0F Christiano. 2020. Learning to summarize with human feedback. Advances in Neural Information Processing Systems 33 (2020) 3008\u20133021."},{"key":"e_1_3_3_3_50_2","unstructured":"Gemma Team Thomas Mesnard Cassidy Hardin Robert Dadashi Surya Bhupatiraju Shreya Pathak Laurent Sifre Morgane Rivi\u00e8re Mihir\u00a0Sanjay Kale Juliette Love et\u00a0al. 2024. Gemma: Open models based on Gemini research and technology. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.08295 (2024)."},{"key":"e_1_3_3_3_51_2","unstructured":"Gemma Team Morgane Riviere Shreya Pathak Pier\u00a0Giuseppe Sessa Cassidy Hardin Surya Bhupatiraju L\u00e9onard Hussenot Thomas Mesnard Bobak Shahriari Alexandre Ram\u00e9 et\u00a0al. 2024. Gemma 2: Improving open language models at a practical size. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2408.00118 (2024)."},{"key":"e_1_3_3_3_52_2","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar et\u00a0al. 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.13971 (2023)."},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.620"},{"key":"e_1_3_3_3_54_2","volume-title":"Advances in Neural Information Processing Systems","author":"Yang Rui","year":"2024","unstructured":"Rui Yang, Ruomeng Ding, Yong Lin, Huan Zhang, and Tong Zhang. 2024. Regularizing Hidden States Enables Learning Generalizable Reward Model for LLMs. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"crossref","unstructured":"Robert\u00a0B Zajonc. 1968. Attitudinal effects of mere exposure. Journal of personality and social psychology 9 2p2 (1968) 1.","DOI":"10.1037\/h0025848"},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"e_1_3_3_3_57_2","unstructured":"Andy Zou Zifan Wang Nicholas Carlini Milad Nasr J\u00a0Zico Kolter and Matt Fredrikson. 2023. Universal and transferable adversarial attacks on aligned language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.15043 (2023)."}],"event":{"name":"FAccT '25: The 2025 ACM Conference on Fairness, Accountability, and Transparency","location":"Athens Greece","acronym":"FAccT '25"},"container-title":["Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3715275.3732068","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T11:18:49Z","timestamp":1750763929000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3715275.3732068"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":56,"alternative-id":["10.1145\/3715275.3732068","10.1145\/3715275"],"URL":"https:\/\/doi.org\/10.1145\/3715275.3732068","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-06-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}