{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T01:16:58Z","timestamp":1765502218261,"version":"3.48.0"},"publisher-location":"New York, NY, USA","reference-count":26,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,10]]},"DOI":"10.1145\/3746252.3760887","type":"proceedings-article","created":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T01:03:42Z","timestamp":1762563822000},"page":"5474-5478","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Uncertainty Quantification for Multiple-Choice Questions is Just One-Token Deep"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8697-2729","authenticated-orcid":false,"given":"Qingcheng","family":"Zeng","sequence":"first","affiliation":[{"name":"Northwestern University, Evanston, IL, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6990-7355","authenticated-orcid":false,"given":"Mingyu","family":"Jin","sequence":"additional","affiliation":[{"name":"Rutgers University, New Brunswick, NJ, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6526-0928","authenticated-orcid":false,"given":"Qinkai","family":"Yu","sequence":"additional","affiliation":[{"name":"University of Exeter, Exeter, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0311-1331","authenticated-orcid":false,"given":"Zhenting","family":"Wang","sequence":"additional","affiliation":[{"name":"Rutgers University, New Brunswick, NJ, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-2043-2704","authenticated-orcid":false,"given":"Wenyue","family":"Hua","sequence":"additional","affiliation":[{"name":"University of California, Santa Barbara, Santa Barbara, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4082-5248","authenticated-orcid":false,"given":"Guangyan","family":"Sun","sequence":"additional","affiliation":[{"name":"University of Rochester, Rochester, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7344-2174","authenticated-orcid":false,"given":"Yanda","family":"Meng","sequence":"additional","affiliation":[{"name":"University of Exeter, Exeter, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1551-8948","authenticated-orcid":false,"given":"Shiqing","family":"Ma","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7570-5756","authenticated-orcid":false,"given":"Qifan","family":"Wang","sequence":"additional","affiliation":[{"name":"Meta AI, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0857-8611","authenticated-orcid":false,"given":"Felix","family":"Juefei-Xu","sequence":"additional","affiliation":[{"name":"Meta AI, New York, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3442-754X","authenticated-orcid":false,"given":"Fan","family":"Yang","sequence":"additional","affiliation":[{"name":"Wake Forest University, Winston-Salem, NC, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6684-6752","authenticated-orcid":false,"given":"Kaize","family":"Ding","sequence":"additional","affiliation":[{"name":"Northwestern University, Evanston, IL, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6476-2336","authenticated-orcid":false,"given":"Ruixiang","family":"Tang","sequence":"additional","affiliation":[{"name":"Rutgers University, New Brunswick, NJ, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2633-8555","authenticated-orcid":false,"given":"Yongfeng","family":"Zhang","sequence":"additional","affiliation":[{"name":"Rutgers University, New Brunswick, NJ, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,11,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"AI@Meta. 2024. Llama 3 Model Card. (2024). https:\/\/github.com\/meta-llama\/llama3\/blob\/main\/MODEL_CARD.md"},{"key":"e_1_3_2_1_2_1","volume-title":"Training Verifiers to Solve Math Word Problems. ArXiv","author":"Cobbe Karl","year":"2021","unstructured":"Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, Christopher Hesse, and John Schulman. 2021. Training Verifiers to Solve Math Word Problems. ArXiv, Vol. abs\/2110.14168 (2021). https:\/\/api.semanticscholar.org\/CorpusID:239998651"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.558"},{"key":"e_1_3_2_1_4_1","volume-title":"Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300","author":"Hendrycks Dan","year":"2020","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2020. Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300 (2020)."},{"key":"e_1_3_2_1_5_1","unstructured":"Dan Hendrycks Collin Burns Steven Basart Andy Zou Mantas Mazeika Dawn Song and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. arXiv:2009.03300 [cs.CY] https:\/\/arxiv.org\/abs\/2009.03300"},{"key":"e_1_3_2_1_6_1","volume-title":"LoRA: Low-Rank Adaptation of Large Language Models. In International Conference on Learning Representations.","author":"Hu Edward J","year":"2021","unstructured":"Edward J Hu, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen, et al., 2021. LoRA: Low-Rank Adaptation of Large Language Models. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1243"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11146421"},{"key":"e_1_3_2_1_9_1","unstructured":"Saurav Kadavath Tom Conerly Amanda Askell Tom Henighan Dawn Drain Ethan Perez Nicholas Schiefer Zac Hatfield-Dodds Nova DasSarma Eli Tran-Johnson et al. 2022. Language models (mostly) know what they know. arXiv preprint arXiv:2207.05221 (2022)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.397"},{"key":"e_1_3_2_1_11_1","unstructured":"Mistral. [n.d.]. Mistral Small 3 | Mistral AI -- mistral.ai. https:\/\/mistral.ai\/news\/mistral-small-3. [Accessed 03-06-2025]."},{"key":"e_1_3_2_1_12_1","unstructured":"Team OLMo Pete Walsh Luca Soldaini Dirk Groeneveld Kyle Lo Shane Arora Akshita Bhagia Yuling Gu Shengyi Huang Matt Jordan Nathan Lambert Dustin Schwenk Oyvind Tafjord Taira Anderson David Atkinson Faeze Brahman Christopher Clark Pradeep Dasigi Nouha Dziri Michal Guerquin Hamish Ivison Pang Wei Koh Jiacheng Liu Saumya Malik William Merrill Lester James V. Miranda Jacob Morrison Tyler Murray Crystal Nam Valentina Pyatkin Aman Rangapur Michael Schmitz Sam Skjonsberg David Wadden Christopher Wilhelm Michael Wilson Luke Zettlemoyer Ali Farhadi Noah A. Smith and Hannaneh Hajishirzi. 2025. 2 OLMo 2 Furious. arXiv:2501.00656 [cs.CL] https:\/\/arxiv.org\/abs\/2501.00656"},{"key":"e_1_3_2_1_13_1","unstructured":"Qwen: An Yang Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chengyuan Li Dayiheng Liu Fei Huang Haoran Wei Huan Lin Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jingren Zhou Junyang Lin Kai Dang Keming Lu Keqin Bao Kexin Yang Le Yu Mei Li Mingfeng Xue Pei Zhang Qin Zhu Rui Men Runji Lin Tianhao Li Tianyi Tang Tingyu Xia Xingzhang Ren Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yu Wan Yuqiong Liu Zeyu Cui Zhenru Zhang and Zihan Qiu. 2025. Qwen2.5 Technical Report. arXiv:2412.15115 [cs.CL] https:\/\/arxiv.org\/abs\/2412.15115"},{"key":"e_1_3_2_1_14_1","volume-title":"Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R. Bowman.","author":"Rein David","year":"2023","unstructured":"David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R. Bowman. 2023. GPQA: A Graduate-Level Google-Proof Q&A Benchmark. arXiv:2311.12022 [cs.AI] https:\/\/arxiv.org\/abs\/2311.12022"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1454"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1099"},{"key":"e_1_3_2_1_17_1","unstructured":"P Team Xinrun Du Yifan Yao Kaijing Ma Bingli Wang Tianyu Zheng King Zhu Minghao Liu Yiming Liang Xiaolong Jin Zhenlin Wei Chujie Zheng Kaixin Deng Shawn Gavin Shian Jia Sichao Jiang Yiyan Liao Rui Li Qinrui Li Sirun Li Yizhi Li Yunwen Li David Ma Yuansheng Ni Haoran Que Qiyao Wang Zhoufutu Wen Siwei Wu Tyshawn Hsing Ming Xu Zhenzhu Yang Zekun Moore Wang Junting Zhou Yuelin Bai Xingyuan Bu Chenglin Cai Liang Chen Yifan Chen Chengtuo Cheng Tianhao Cheng Keyi Ding Siming Huang Yun Huang Yaoru Li Yizhe Li Zhaoqun Li Tianhao Liang Chengdong Lin Hongquan Lin Yinghao Ma Tianyang Pang Zhongyuan Peng Zifan Peng Qige Qi Shi Qiu Xingwei Qu Shanghaoran Quan Yizhou Tan Zili Wang Chenqing Wang Hao Wang Yiya Wang Yubo Wang Jiajun Xu Kexin Yang Ruibin Yuan Yuanhao Yue Tianyang Zhan Chun Zhang Jinyang Zhang Xiyue Zhang Xingjian Zhang Yue Zhang Yongchi Zhao Xiangyu Zheng Chenghua Zhong Yang Gao Zhoujun Li Dayiheng Liu Qian Liu Tianyu Liu Shiwen Ni Junran Peng Yujia Qin Wenbo Su Guoyin Wang Shi Wang Jian Yang Min Yang Meng Cao Xiang Yue Zhaoxiang Zhang Wangchunshu Zhou Jiaheng Liu Qunshu Lin Wenhao Huang and Ge Zhang. 2025. SuperGPQA: Scaling LLM Evaluation across 285 Graduate Disciplines. arXiv:2502.14739 [cs.CL] https:\/\/arxiv.org\/abs\/2502.14739"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.330"},{"key":"e_1_3_2_1_19_1","volume-title":"Kirill Grishchenkov, Sergey Petrakov, Alexander Panchenko, Timothy Baldwin, Preslav Nakov, Maxim Panov, and Artem Shelmanov.","author":"Vashurin Roman","year":"2024","unstructured":"Roman Vashurin, Ekaterina Fadeeva, Artem Vazhentsev, Lyudmila Rvanova, Akim Tsvigun, Daniil Vasilev, Rui Xing, Abdelrahman Boda Sadallah, Kirill Grishchenkov, Sergey Petrakov, Alexander Panchenko, Timothy Baldwin, Preslav Nakov, Maxim Panov, and Artem Shelmanov. 2024. Benchmarking Uncertainty Quantification Methods for Large Language Models with LM-Polygraph. arXiv preprint arXiv:2406.15627 (2024)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.441"},{"key":"e_1_3_2_1_21_1","unstructured":"Yubo Wang Xueguang Ma Ge Zhang Yuansheng Ni Abhranil Chandra Shiguang Guo Weiming Ren Aaran Arulraj Xuan He Ziyan Jiang Tianle Li Max Ku Kai Wang Alex Zhuang Rongqi Fan Xiang Yue and Wenhu Chen. 2024b. MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark. arXiv:2406.01574 [cs.CL] https:\/\/arxiv.org\/abs\/2406.01574"},{"key":"e_1_3_2_1_22_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2023","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed Chi, Quoc Le, and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arXiv:2201.11903"},{"key":"e_1_3_2_1_23_1","unstructured":"Miao Xiong Zhiyuan Hu Xinyang Lu Yifei Li Jie Fu Junxian He and Bryan Hooi. 2024. Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs. arXiv:2306.13063 [cs.CL] https:\/\/arxiv.org\/abs\/2306.13063"},{"key":"e_1_3_2_1_24_1","volume-title":"Benchmarking LLMs via Uncertainty Quantification. arXiv preprint arXiv:2401.12794","author":"Ye Fanghua","year":"2024","unstructured":"Fanghua Ye, Yang MingMing, Jianhui Pang, Longyue Wang, Derek F Wong, Yilmaz Emine, Shuming Shi, and Zhaopeng Tu. 2024. Benchmarking LLMs via Uncertainty Quantification. arXiv preprint arXiv:2401.12794 (2024)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1472"},{"key":"e_1_3_2_1_26_1","unstructured":"Chujie Zheng Hao Zhou Fandong Meng Jie Zhou and Minlie Huang. 2024. Large Language Models Are Not Robust Multiple Choice Selectors. arXiv:2309.03882 [cs.CL] https:\/\/arxiv.org\/abs\/2309.03882"}],"event":{"name":"CIKM '25: The 34th ACM International Conference on Information and Knowledge Management","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Seoul Republic of Korea","acronym":"CIKM '25"},"container-title":["Proceedings of the 34th ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3746252.3760887","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T01:15:06Z","timestamp":1765502106000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3746252.3760887"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,10]]},"references-count":26,"alternative-id":["10.1145\/3746252.3760887","10.1145\/3746252"],"URL":"https:\/\/doi.org\/10.1145\/3746252.3760887","relation":{},"subject":[],"published":{"date-parts":[[2025,11,10]]},"assertion":[{"value":"2025-11-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}