{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T01:01:26Z","timestamp":1774400486509,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657860","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"819-829","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["TREC iKAT 2023: A Test Collection for Evaluating Conversational and Interactive Knowledge Assistants"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9447-4172","authenticated-orcid":false,"given":"Mohammad","family":"Aliannejadi","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amstedam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4046-3419","authenticated-orcid":false,"given":"Zahra","family":"Abbasiantaeb","sequence":"additional","affiliation":[{"name":"IVI, University of Amsterdam, Amsterdam, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6729-1346","authenticated-orcid":false,"given":"Shubham","family":"Chatterjee","sequence":"additional","affiliation":[{"name":"University of Edinburgh, Edinburgh, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2422-8651","authenticated-orcid":false,"given":"Jeffrey","family":"Dalton","sequence":"additional","affiliation":[{"name":"University of Edinburgh, Edinburgh, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6900-0557","authenticated-orcid":false,"given":"Leif","family":"Azzopardi","sequence":"additional","affiliation":[{"name":"Computer and Information Sciences, University of Strathclyde, Glasgow, United Kingdom"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Zahra Abbasiantaeb and Mohammad Aliannejadi. 2024. Generate then Retrieve: Conversational Response Retrieval Using LLMs as Answer and Query Generators. arxiv: 2403.19302 [cs.IR]"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3616855.3635856"},{"key":"e_1_3_2_1_3_1","volume-title":"Text REtrieval Conference (TREC). NIST.","author":"Aliannejadi Mohammad","year":"2024","unstructured":"Mohammad Aliannejadi, Zahra Abbasiantaeb, Shubham Chatterjee, Jeffery Dalton, and Leif Azzopardi. 2024. TREC iKAT 2023: The Interactive Knowledge Assistance Track Overview. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_4_1","unstructured":"Leif Azzopardi Mateusz Dubiel Martin Halvey and Jeffery Dalton. 2018. Conceptualizing agent-human interactions during the conversational search process. In The second international workshop on conversational approaches to information retrieval."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1394251.1394261"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-238.interactive-RutgersB"},{"key":"e_1_3_2_1_7_1","volume-title":"Text REtrieval Conference (TREC). NIST.","author":"Dalton Jeffrey","year":"2020","unstructured":"Jeffrey Dalton, Chenyan Xiong, and Jamie Callan. 2020a. CAsT 2019: The conversational assistance track overview. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_8_1","volume-title":"Text REtrieval Conference (TREC). NIST.","author":"Dalton Jeffrey","year":"2021","unstructured":"Jeffrey Dalton, Chenyan Xiong, and Jamie Callan. 2021. TREC CAsT 2021: The Conversational Assistance Track Overview. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401206"},{"key":"e_1_3_2_1_10_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arxiv","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arxiv: 1810.04805 [cs.CL]"},{"key":"e_1_3_2_1_11_1","unstructured":"Zishan Guo Renren Jin Chuang Liu Yufei Huang Dan Shi Supryadi Linhao Yu Yan Liu Jiaxuan Li Bojian Xiong and Deyi Xiong. 2023. Evaluating Large Language Models: A Comprehensive Survey. arxiv: 2310.19736 [cs.CL]"},{"key":"e_1_3_2_1_12_1","volume-title":"TREC 2002 Interactive Track Report. In Text REtrieval Conference (TREC), Ellen M. Voorhees and Lori P. Buckland (Eds.). NIST.","author":"Hersh William R.","year":"2002","unstructured":"William R. Hersh. 2002. TREC 2002 Interactive Track Report. In Text REtrieval Conference (TREC), Ellen M. Voorhees and Lori P. Buckland (Eds.). NIST."},{"key":"e_1_3_2_1_13_1","volume-title":"Shibani Santurkar, Surya Ganguli, Tatsunori Hashimoto, Thomas Icard, Tianyi Zhang, Vishrav Chaudhary, William Wang, Xuechen Li, Yifan Mai, Yuhui Zhang, and Yuta Koreeda.","author":"Liang Percy","year":"2023","unstructured":"Percy Liang, Rishi Bommasani, Tony Lee, Dimitris Tsipras, Dilara Soylu, Michihiro Yasunaga, Yian Zhang, Deepak Narayanan, Yuhuai Wu, Ananya Kumar, Benjamin Newman, Binhang Yuan, Bobby Yan, Ce Zhang, Christian Cosgrove, Christopher D. Manning, Christopher R\u00e9, Diana Acosta-Navas, Drew A. Hudson, Eric Zelikman, Esin Durmus, Faisal Ladhak, Frieda Rong, Hongyu Ren, Huaxiu Yao, Jue Wang, Keshav Santhanam, Laurel Orr, Lucia Zheng, Mert Yuksekgonul, Mirac Suzgun, Nathan Kim, Neel Guha, Niladri Chatterji, Omar Khattab, Peter Henderson, Qian Huang, Ryan Chi, Sang Michael Xie, Shibani Santurkar, Surya Ganguli, Tatsunori Hashimoto, Thomas Icard, Tianyi Zhang, Vishrav Chaudhary, William Wang, Xuechen Li, Yifan Mai, Yuhui Zhang, and Yuta Koreeda. 2023. Holistic Evaluation of Language Models. arxiv: 2211.09110 [cs.CL]"},{"key":"e_1_3_2_1_14_1","volume-title":"Nafise Sadat Moosavi, and Chenghua Lin","author":"Liu Yiqi","year":"2023","unstructured":"Yiqi Liu, Nafise Sadat Moosavi, and Chenghua Lin. 2023. LLMs as Narcissistic Evaluators: When Ego Inflates Evaluation Scores. arxiv: 2311.09766 [cs.CL]"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00053-4"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Arnold Overwijk Chenyan Xiong Xiao Liu Cameron VandenBerg and Jamie Callan. 2022. ClueWeb22: 10 Billion Web Documents with Visual and Semantic Information. arxiv: 2211.15848 [cs.IR]","DOI":"10.1145\/3477495.3536321"},{"key":"e_1_3_2_1_17_1","volume-title":"Text REtrieval Conference (TREC). NIST.","author":"Owoicho Paul","year":"2023","unstructured":"Paul Owoicho, Jeffrey Dalton, Mohammad Aliannejadi, Leif Azzopardi, Johanne R Trippas, and Svitlana Vakulenko. 2023. TREC CAsT 2022: Going Beyond User Ask and System Retrieve with Initiative and Response Generation. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3020165.3020183"},{"key":"e_1_3_2_1_19_1","volume-title":"Findings of the Association for Computational Linguistics (EACL)","author":"Rahmani Hossein A.","unstructured":"Hossein A. Rahmani, Xi Wang, Mohammad Aliannejadi, Mohammadmehdi Naghiaei, and Emine Yilmaz. 2024. Clarifying the Path to User Satisfaction: An Investigation into Clarification Usefulness. In Findings of the Association for Computational Linguistics (EACL). Association for Computational Linguistics, 1266--1277."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Pranav Rajpurkar Jian Zhang Konstantin Lopyrev and Percy Liang. 2016. SQuAD: 100 000 Questions for Machine Comprehension of Text. arxiv: 1606.05250 [cs.CL]","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_21_1","volume-title":"Manning","author":"Reddy Siva","year":"2019","unstructured":"Siva Reddy, Danqi Chen, and Christopher D. Manning. 2019. CoQA: A Conversational Question Answering Challenge. arxiv: 1808.07042 [cs.CL]"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2018.07.003"},{"key":"e_1_3_2_1_23_1","unstructured":"Aarohi Srivastava Abhinav Rastogi Abhishek Rao Abu Awal Md Shoeb Abubakar Abid Adam Fisch Adam R. Brown Adam Santoro Aditya Gupta Adri\u00e0 Garriga-Alonso Agnieszka Kluska Aitor Lewkowycz Akshat Agarwal Alethea Power Alex Ray Alex Warstadt Alexander W. Kocurek Ali Safaya Ali Tazarv Alice"},{"key":"e_1_3_2_1_24_1","unstructured":"Romal Thoppilan Daniel De Freitas Jamie Hall Noam Shazeer Apoorv Kulshreshtha Heng-Tze Cheng Alicia Jin Taylor Bos Leslie Baker Yu Du YaGuang Li Hongrae Lee Huaixiu Steven Zheng Amin Ghafouri Marcelo Menegali Yanping Huang Maxim Krikun Dmitry Lepikhin James Qin Dehao Chen Yuanzhong Xu Zhifeng Chen Adam Roberts Maarten Bosma Vincent Zhao Yanqi Zhou Chung-Ching Chang Igor Krivokon Will Rusch Marc Pickett Pranesh Srinivasan Laichee Man Kathleen Meier-Hellstern Meredith Ringel Morris Tulsee Doshi Renelito Delos Santos Toju Duke Johnny Soraker Ben Zevenbergen Vinodkumar Prabhakaran Mark Diaz Ben Hutchinson Kristen Olson Alejandra Molina Erin Hoffman-John Josh Lee Lora Aroyo Ravi Rajakumar Alena Butryna Matthew Lamm Viktoriya Kuzmina Joe Fenton Aaron Cohen Rachel Bernstein Ray Kurzweil Blaise Aguera-Arcas Claire Cui Marian Croak Ed Chi and Quoc Le. 2022. LaMDA: Language Models for Dialog Applications. arxiv: 2201.08239 [cs.CL]"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3466796"},{"key":"e_1_3_2_1_26_1","volume-title":"Bowman","author":"Wang Alex","year":"2020","unstructured":"Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R. Bowman. 2020. SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems. arxiv: 1905.00537 [cs.CL]"},{"key":"e_1_3_2_1_27_1","volume-title":"TREC 2016 Dynamic Domain Track Overview.. In Text REtrieval Conference (TREC). NIST.","author":"Yang Grace Hui","year":"2016","unstructured":"Grace Hui Yang and Ian Soboroff. 2016. TREC 2016 Dynamic Domain Track Overview.. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_28_1","volume-title":"TREC 2017 Dynamic Domain Track Overview. In Text REtrieval Conference (TREC). NIST.","author":"Yang Grace Hui","year":"2017","unstructured":"Grace Hui Yang, Zhiwen Tang, and Ian Soboroff. 2017. TREC 2017 Dynamic Domain Track Overview. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_29_1","volume-title":"TREC 2015 Dynamic Domain Track Overview. In Text REtrieval Conference (TREC). NIST.","author":"Yang Hui","year":"2015","unstructured":"Hui Yang, John R. Frank, and Ian Soboroff. 2015. TREC 2015 Dynamic Domain Track Overview. In Text REtrieval Conference (TREC). NIST."},{"key":"e_1_3_2_1_30_1","volume-title":"HellaSwag: Can a Machine Really Finish Your Sentence?arxiv","author":"Zellers Rowan","year":"1905","unstructured":"Rowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi. 2019. HellaSwag: Can a Machine Really Finish Your Sentence?arxiv: 1905.07830 [cs.CL]"},{"key":"e_1_3_2_1_31_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric. P Xing Hao Zhang Joseph E. Gonzalez and Ion Stoica. 2023. Judging LLM-as-a-judge with MT-Bench and Chatbot Arena. arxiv: 2306.05685 [cs.CL]"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657860","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657860","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:29:41Z","timestamp":1755840581000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657860"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":31,"alternative-id":["10.1145\/3626772.3657860","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657860","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}