{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:07:28Z","timestamp":1755907648469,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":181,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100006374","name":"Australian Research Council","doi-asserted-by":"publisher","award":["FT240100022, CE200100005"],"award-info":[{"award-number":["FT240100022, CE200100005"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CPS-243893"],"award-info":[{"award-number":["CPS-243893"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,18]]},"DOI":"10.1145\/3731120.3744612","type":"proceedings-article","created":{"date-parts":[[2025,7,18]],"date-time":"2025-07-18T13:34:06Z","timestamp":1752845646000},"page":"78-91","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Preaching to the ChoIR: Lessons IR Should Share with AI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7311-3693","authenticated-orcid":false,"given":"Gianluca","family":"Demartini","sequence":"first","affiliation":[{"name":"The U. of Queensland, Brisbane, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9879-6470","authenticated-orcid":false,"given":"Claudia","family":"Hauff","sequence":"additional","affiliation":[{"name":"Spotify, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0056-2834","authenticated-orcid":false,"given":"Matthew","family":"Lease","sequence":"additional","affiliation":[{"name":"The U. of Texas at Austin, Austin, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2852-168X","authenticated-orcid":false,"given":"Stefano","family":"Mizzaro","sequence":"additional","affiliation":[{"name":"University of Udine, Udine, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9191-3280","authenticated-orcid":false,"given":"Kevin","family":"Roitero","sequence":"additional","affiliation":[{"name":"University of Udine, Udine, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0487-9609","authenticated-orcid":false,"given":"Mark","family":"Sanderson","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9094-0810","authenticated-orcid":false,"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3797-4293","authenticated-orcid":false,"given":"Chirag","family":"Shah","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, WA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9913-433X","authenticated-orcid":false,"given":"Damiano","family":"Spina","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2425-3136","authenticated-orcid":false,"given":"Paul","family":"Thomas","sequence":"additional","affiliation":[{"name":"Microsoft, Adelaide, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2888-4202","authenticated-orcid":false,"given":"Arjen P.","family":"de Vries","sequence":"additional","affiliation":[{"name":"Radboud University, Nijmegen, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0271-5563","authenticated-orcid":false,"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[{"name":"The U. of Queensland, Brisbane, Australia"}]}],"member":"320","published-online":{"date-parts":[[2025,7,18]]},"reference":[{"volume-title":"Artificial intelligence - Wikipedia. https:\/\/en.wikipedia.org\/wiki\/Artificial_intelligence. Accessed","year":"2025","key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. Artificial intelligence - Wikipedia. https:\/\/en.wikipedia.org\/wiki\/Artificial_intelligence. Accessed: 17 Feb 2025."},{"volume-title":"Common Corpus - Hugging Face. https:\/\/huggingface.co\/datasets\/PleIAs\/common_corpus. Accessed","year":"2025","key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. Common Corpus - Hugging Face. https:\/\/huggingface.co\/datasets\/PleIAs\/common_corpus. Accessed: 17 Feb 2025."},{"volume-title":"https:\/\/www.elastic.co\/elasticsearch. Accessed","year":"2025","key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. Elasticsearch. https:\/\/www.elastic.co\/elasticsearch. Accessed: 17 Feb 2025."},{"volume-title":"Swirl AI Search. https:\/\/swirlaiconnect.com\/. Accessed","year":"2025","key":"e_1_3_2_1_4_1","unstructured":"[n.d.]. Swirl AI Search. https:\/\/swirlaiconnect.com\/. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_5_1","volume-title":"AI Employment Outlook: Salary Trends and Future Projections. https:\/\/www.aitimejournal.com\/ai-employment-outlook-salarytrends-and-future-projections\/46785\/. Accessed","author":"Times Journal AI","year":"2025","unstructured":"AI Times Journal. 2023. AI Employment Outlook: Salary Trends and Future Projections. https:\/\/www.aitimejournal.com\/ai-employment-outlook-salarytrends-and-future-projections\/46785\/. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_6_1","unstructured":"James Allan Eunsol Choi Daniel P Lopresti and Hamed Zamani. 2024. Future of Information Retrieval Research in the Age of Generative AI. Technical Report. https:\/\/cra.org\/wp-content\/uploads\/2024\/12\/Future-of-Information-Retrieval-Research-in-the-Age-of-Generative-AI.pdf"},{"key":"e_1_3_2_1_7_1","volume-title":"Evaluating the Evaluations: A Perspective on Benchmarks. ACM SIGIR Forum 58, 2","author":"Alonso Omar","year":"2024","unstructured":"Omar Alonso and Kenneth Church. 2024. Evaluating the Evaluations: A Perspective on Benchmarks. ACM SIGIR Forum 58, 2 (2024). https:\/\/sigir.org\/wpcontent\/uploads\/2024\/02\/p18.pdf Opinion Paper."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-020-09376-y"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"volume-title":"The Information Manifold: Why Computers Can't Solve Algorithmic Bias and Fake News","author":"Badia Antonio","key":"e_1_3_2_1_11_1","unstructured":"Antonio Badia. 2019. The Information Manifold: Why Computers Can't Solve Algorithmic Bias and Fake News. MIT Press, Cambridge, MA. xvii 334 pages."},{"key":"e_1_3_2_1_12_1","unstructured":"Jinze Bai Shuai Bai Yunfei Chu Zeyu Cui Kai Dang Xiaodong Deng Yang Fan Wenbin Ge Yu Han Fei Huang Binyuan Hui Luo Ji Mei Li Junyang Lin Runji Lin Dayiheng Liu Gao Liu Chengqiang Lu Keming Lu Jianxin Ma Rui Men Xingzhang Ren Xuancheng Ren Chuanqi Tan Sinan Tan Jianhong Tu Peng Wang Shijie Wang Wei Wang Shengguang Wu Benfeng Xu Jin Xu An Yang Hao Yang Jian Yang Shusheng Yang Yang Yao Bowen Yu Hongyi Yuan Zheng Yuan Jianwei Zhang Xingxuan Zhang Yichang Zhang Zhenru Zhang Chang Zhou Jingren Zhou Xiaohuan Zhou and Tianhang Zhu. 2023. Qwen Technical Report. arXiv preprint arXiv:2309.16609 (2023)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914671"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080839"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","unstructured":"Sourav Banerjee Ayushi Agarwal and Eishkaran Singh. 2024. The Vulnerability of Language Model Benchmarks: Do They Accurately Reflect True LLM Performance? arXiv. doi:10.48550\/arXiv.2412.03597","DOI":"10.48550\/arXiv.2412.03597"},{"key":"e_1_3_2_1_16_1","unstructured":"Michael Barbaro and Tom Zeller Jr. 2006. A Face is Exposed for AOL Searcher No. 4417749. https:\/\/www.nytimes.com\/2006\/08\/09\/technology\/09aol.html."},{"key":"e_1_3_2_1_17_1","volume-title":"Alexa Tells 10-year-old Girl to Touch Live Plug With Penny. https: \/\/www.bbc.com\/news\/technology-59810383. Accessed","author":"BBC.","year":"2025","unstructured":"BBC. 2021. Alexa Tells 10-year-old Girl to Touch Live Plug With Penny. https: \/\/www.bbc.com\/news\/technology-59810383. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2043932.2043996"},{"key":"e_1_3_2_1_19_1","article-title":"A neural probabilistic language model","author":"Bengio Yoshua","year":"2003","unstructured":"Yoshua Bengio, R\u00e9jean Ducharme, Pascal Vincent, and Christian Janvin. 2003. A neural probabilistic language model. J. Mach. Learn. Res. 3, null (March 2003), 1137-1155.","journal-title":"J. Mach. Learn. Res. 3, null"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/3495724.3495883"},{"key":"e_1_3_2_1_21_1","volume-title":"Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang.","author":"Bubeck S\u00e9bastien","year":"2023","unstructured":"S\u00e9bastien Bubeck, Varun Chandrasekaran, Ronen Eldan, Johannes Gehrke, Eric Horvitz, Ece Kamar, Peter Lee, Yin Tat Lee, Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang. 2023. Sparks of Artificial General Intelligence: Early experiments with GPT-4. arXiv:2303.12712 [cs.CL] https:\/\/arxiv.org\/abs\/2303.12712"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345543"},{"key":"e_1_3_2_1_23_1","unstructured":"Patrick Butlin Robert Long Eric Elmoznino Yoshua Bengio Jonathan Birch Axel Constant George Deane Stephen M. Fleming Chris Frith Xu Ji Ryota Kanai Colin Klein Grace Lindsay Matthias Michel Liad Mudrik Megan A. K. Peters Eric Schwitzgebel Jonathan Simon and Rufin VanRullen. 2023. Consciousness in Artificial Intelligence: Insights from the Science of Consciousness. arXiv:2308.08708 [cs.AI] https:\/\/arxiv.org\/abs\/2308.08708"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148219"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Carlos Castillo Brian D Davison et al. 2011. Adversarialweb search. Foundations and trends\u00ae in information retrieval 4 5 (2011) 377-486.","DOI":"10.1561\/1500000021"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1365815.1365816"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_1_29_1","unstructured":"Eugene Charniak. 1985. Introduction to artificial intelligence. Pearson Education India."},{"key":"e_1_3_2_1_30_1","volume-title":"Decision transformer: Reinforcement learning via sequence modeling. Advances in neural information processing systems 34","author":"Chen Lili","year":"2021","unstructured":"Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Misha Laskin, Pieter Abbeel, Aravind Srinivas, and Igor Mordatch. 2021. Decision transformer: Reinforcement learning via sequence modeling. Advances in neural information processing systems 34 (2021), 15084-15097."},{"key":"e_1_3_2_1_31_1","volume-title":"Uprise: Universal prompt retrieval for improving zero-shot evaluation. arXiv preprint arXiv:2303.08518","author":"Cheng Daixuan","year":"2023","unstructured":"Daixuan Cheng, Shaohan Huang, Junyu Bi, Yuefeng Zhan, Jianfeng Liu, Yujing Wang, Hao Sun, Furu Wei, Denvy Deng, and Qi Zhang. 2023. Uprise: Universal prompt retrieval for improving zero-shot evaluation. arXiv preprint arXiv:2303.08518 (2023)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","unstructured":"Peter Clark Isaac Cowhey Oren Etzioni Tushar Khot Ashish Sabharwal Carissa Schoenick and Oyvind Tafjord. 2018. Think you have Solved Question Answering? Try ARC the AI2 Reasoning Challenge. arXiv. doi:10.48550\/arXiv.1803. 05457 arXiv:1803.05457 [cs.AI]","DOI":"10.48550\/arXiv.1803"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657834"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3712001"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1327452.1327492"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/1863879.1863883"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0184604"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1423"},{"key":"e_1_3_2_1_39_1","volume-title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. CoRR abs\/2010.11929","author":"Dosovitskiy Alexey","year":"2020","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, and Neil Houlsby. 2020. An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. CoRR abs\/2010.11929 (2020). arXiv:2010.11929 https:\/\/arxiv.org\/abs\/2010.11929"},{"key":"e_1_3_2_1_40_1","unstructured":"Aaron Grattafiori et al. 2024. The Llama 3 Herd of Models. arXiv:2407.21783 [cs.AI] https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2024.106714"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3116857"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"Marco Ferrante Nicola Ferro and Norbert Fuhr. 2022. Response to Moffat's Comment on ''Towards Meaningful Statements in IR Evaluation: Mapping Evaluation Measures to Interval Scales''. arXiv:2212.11735 [cs.IR] https:\/\/arxiv.org\/abs\/2212.11735","DOI":"10.1109\/ACCESS.2021.3116857"},{"volume-title":"Information Retrieval Evaluation in a Changing World Lessons Learned from 20 Years of CLEF","author":"Ferro Nicola","key":"e_1_3_2_1_44_1","unstructured":"Nicola Ferro and Carol Peters. 2019. Information Retrieval Evaluation in a Changing World Lessons Learned from 20 Years of CLEF. Springer."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-024-09670-4"},{"key":"e_1_3_2_1_46_1","unstructured":"J\u00f6rg Frohberg and Frank Binder. 2022. CRASS: A Novel Data Set and Benchmark to Test Counterfactual Reasoning of Large Language Models. In Proceedings of the Thirteenth Language Resources and Evaluation Conference Nicoletta Calzolari Fr\u00e9d\u00e9ric B\u00e9chet Philippe Blache Khalid Choukri Christopher Cieri Thierry Declerck Sara Goggi Hitoshi Isahara Bente Maegaard Joseph Mariani H\u00e9l\u00e8ne Mazo Jan Odijk and Stelios Piperidis (Eds.). European Language Resources Association Marseille France 2126-2140. https:\/\/aclanthology.org\/2022.lrec-1.229\/"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3632754"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658900"},{"key":"e_1_3_2_1_50_1","volume-title":"The Information: A History, a Theory, a Flood","author":"Gleick James","year":"2011","unstructured":"James Gleick. 2011. The Information: A History, a Theory, a Flood. Pantheon."},{"key":"e_1_3_2_1_51_1","volume-title":"Perspectives on the State and Future of Deep Learning -","author":"Goldblum Micah","year":"2023","unstructured":"Micah Goldblum, Anima Anandkumar, Richard Baraniuk, Tom Goldstein, Kyunghyun Cho, Zachary C Lipton, Melanie Mitchell, Preetum Nakkiran, Max Welling, and Andrew Gordon Wilson. 2023. Perspectives on the State and Future of Deep Learning - 2023. arXiv:2312.09323 [cs.AI] https:\/\/arxiv.org\/abs\/2312.09323"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1177\/09637214241262329"},{"key":"e_1_3_2_1_53_1","volume-title":"Hamish Ivison, Ian Magnusson, Yizhong Wang, et al.","author":"Groeneveld Dirk","year":"2024","unstructured":"Dirk Groeneveld, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, et al. 2024. Olmo: Accelerating the science of language models. arXiv preprint arXiv:2402.00838 (2024)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/1629096.1629099"},{"key":"e_1_3_2_1_55_1","volume-title":"Connecting large language models with evolutionary algorithms yields powerful prompt optimizers. arXiv preprint arXiv:2309.08532","author":"Guo Qingyan","year":"2023","unstructured":"Qingyan Guo, Rui Wang, Junliang Guo, Bei Li, Kaitao Song, Xu Tan, Guoqing Liu, Jiang Bian, and Yujiu Yang. 2023. Connecting large language models with evolutionary algorithms yields powerful prompt optimizers. arXiv preprint arXiv:2309.08532 (2023)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000065"},{"volume-title":"Artificial intelligence: The very idea","author":"Haugeland John","key":"e_1_3_2_1_57_1","unstructured":"John Haugeland. 1989. Artificial intelligence: The very idea. MIT press."},{"key":"e_1_3_2_1_58_1","volume-title":"Does Prompt Formatting Have Any Impact on LLM Performance? arXiv preprint arXiv:2411.10541","author":"He Jia","year":"2024","unstructured":"Jia He, Mukund Rungta, David Koleczek, Arshdeep Sekhon, Franklin X Wang, and Sadid Hasan. 2024. Does Prompt Formatting Have Any Impact on LLM Performance? arXiv preprint arXiv:2411.10541 (2024)."},{"key":"e_1_3_2_1_59_1","volume-title":"Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. Open-Review.net, Online, 1-27","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations. Open-Review.net, Online, 1-27. https:\/\/openreview.net\/pdf?id=d7KBjmI3GmQ"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3637866"},{"key":"e_1_3_2_1_62_1","volume-title":"Mitchell","author":"Horvitz Eric","year":"2024","unstructured":"Eric Horvitz and Tom M. Mitchell. 2024. Scientific Progress in Artificial Intelligence: History, Status, and Futures. In Realizing the Promise and Minimizing the Perils of AI for Science and the Scientific Community, Kathleen Hall Jamieson, Anne-Marie Mazza, and William Kearney (Eds.). University of Pennsylvania Press."},{"volume-title":"AI Engineering: Building Applications with Foundation Models","author":"Huyen Chip","key":"e_1_3_2_1_63_1","unstructured":"Chip Huyen. 2024. AI Engineering: Building Applications with Foundation Models. O'Reilly Media."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0088-2"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","unstructured":"M. I. Jordan and T. M. Mitchell. 2015. Machine learning: Trends perspectives and prospects. Science 349 6245 (2015) 255-260. doi:10.1126\/science.aaa8415 arXiv:https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aaa8415","DOI":"10.1126\/science.aaa8415"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"crossref","unstructured":"John Jumper Richard Evans Alexander Pritzel Tim Green Michael Figurnov Olaf Ronneberger Kathryn Tunyasuvunakool Russ Bates Augustin \u017d\u00eddek Anna Potapenko et al. 2021. Highly accurate protein structure prediction with AlphaFold. nature 596 7873 (2021) 583-589.","DOI":"10.1038\/s41586-021-03819-2"},{"key":"e_1_3_2_1_67_1","volume-title":"International journal of innovative science and research technology 8, 3","author":"Kalla Dinesh","year":"2023","unstructured":"Dinesh Kalla, Nathan Smith, Fnu Samaah, and Sivaraju Kuraku. 2023. Study and analysis of chat GPT and its impact on different fields of study. International journal of innovative science and research technology 8, 3 (2023)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2882782"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/1629175.1629210"},{"key":"e_1_3_2_1_70_1","volume-title":"Advances in Neural Information Processing Systems","author":"Kim Jaehee","year":"2024","unstructured":"Jaehee Kim, Yukyung Lee, and Pilsung Kang. 2024. A Gradient Accumulation Method for Dense Retriever under Memory Constraint. In Advances in Neural Information Processing Systems, A. Globerson, L. Mackey, D. Belgrave, A. Fan, U. Paquet, J. Tomczak, and C. Zhang (Eds.), Vol. 37. Curran Associates, Inc., 11765-11788. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2024\/file\/15ba84c1e19b0eb75f96922f5da0a021-Paper-Conference.pdf"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23959"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1082"},{"key":"e_1_3_2_1_74_1","volume-title":"Alisa Liu, Nouha Dziri, Shane Lyu, et al.","author":"Lambert Nathan","year":"2024","unstructured":"Nathan Lambert, Jacob Morrison, Valentina Pyatkin, Shengyi Huang, Hamish Ivison, Faeze Brahman, Lester James V Miranda, Alisa Liu, Nouha Dziri, Shane Lyu, et al. 2024. T'' ulu 3: Pushing frontiers in open language model posttraining. arXiv preprint arXiv:2411.15124 (2024)."},{"key":"e_1_3_2_1_75_1","volume-title":"Godfather of AI' predicts it will take over the world. https:\/\/youtu.be\/vxkBE23zDmQ. Accessed","author":"LBC.","year":"2025","unstructured":"LBC. 2025. 'Godfather of AI' predicts it will take over the world. https:\/\/youtu.be\/vxkBE23zDmQ. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/1316874.1316876"},{"key":"e_1_3_2_1_77_1","volume-title":"Deep learning. nature 521, 7553","author":"LeCun Yann","year":"2015","unstructured":"Yann LeCun, Yoshua Bengio, and Geoffrey Hinton. 2015. Deep learning. nature 521, 7553 (2015), 436-444."},{"key":"e_1_3_2_1_78_1","first-page":"9459","volume-title":"Lin (Eds.)","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis,Wen-tau Yih, Tim Rockt\u00e4schel, Sebastian Riedel, and Douwe Kiela. 2020. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (Eds.), Vol. 33. Curran Associates, Inc., 9459-9474. https:\/\/proceedings.neurips. cc\/paper_files\/paper\/2020\/file\/6b493230205f780e1bc26945df7481e5-Paper.pdf"},{"key":"e_1_3_2_1_79_1","volume-title":"Gathering Strength","author":"Littman Michael L.","year":"2021","unstructured":"Michael L. Littman, Ifeoma Ajunwa, Guy Berger, Craig Boutilier, Morgan Currie, Finale Doshi-Velez, Gillian Hadfield, Michael C. Horowitz, Charles Isbell, Hiroaki Kitano, Karen Levy, Terah Lyons, Melanie Mitchell, Julie Shah, Steven Sloman, Shannon Vallor, and Toby Walsh. 2021. Gathering Strength, Gathering Storms: The One Hundred Year Study on Artificial Intelligence (AI100) 2021 Study Panel Report. Technical Report. Stanford University, Stanford, CA. http:\/\/ai100.stanford.edu\/2021-report"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1080\/23270012.2019.1570365"},{"key":"e_1_3_2_1_81_1","volume-title":"AI Index: State of AI in 13 Charts. https:\/\/hai.stanford.edu\/news\/ai-index-state-ai-13-charts. Accessed","author":"Lynch Shana","year":"2025","unstructured":"Shana Lynch. 2024. AI Index: State of AI in 13 Charts. https:\/\/hai.stanford.edu\/news\/ai-index-state-ai-13-charts. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_82_1","volume-title":"d.]. Marcus on AI. https:\/\/garymarcus.substack.com. Accessed","author":"Marcus Gary","year":"2025","unstructured":"Gary Marcus. [n. d.]. Marcus on AI. https:\/\/garymarcus.substack.com. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_83_1","volume-title":"Deep Learning: A Critical Appraisal. CoRR abs\/1801.00631","author":"Marcus Gary","year":"2018","unstructured":"Gary Marcus. 2018. Deep Learning: A Critical Appraisal. CoRR abs\/1801.00631 (2018). arXiv:1801.00631 http:\/\/arxiv.org\/abs\/1801.00631"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00339-6"},{"key":"e_1_3_2_1_85_1","volume-title":"Yoav Shoham, Russell Wald, Tobi Walsh, Armin Hamrah, Lapo Santarlasci, Julia Betts Lotufo, Alexandra Rome, Andrew Shi, and Sukrut Oak.","author":"Maslej Nestor","year":"2025","unstructured":"Nestor Maslej, Loredana Fattorini, Raymond Perrault, Yolanda Gil, Vanessa Parli, Njenga Kariuki, Emily Capstick, Anka Reuel, Erik Brynjolfsson, John Etchemendy, Katrina Ligett, Terah Lyons, James Manyika, Juan Carlos Niebles, Yoav Shoham, Russell Wald, Tobi Walsh, Armin Hamrah, Lapo Santarlasci, Julia Betts Lotufo, Alexandra Rome, Andrew Shi, and Sukrut Oak. 2025. The AI Index 2025 Annual Report. Technical Report. AI Index Steering Committee, Institute for Human-Centered AI, Stanford University, Stanford, CA. https:\/\/hai.stanford.edu\/assets\/files\/hai_ai_index_report_2025.pdf"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/1045339.1045340"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-024-00897-5"},{"volume-title":"d.]. Llama AI Model. https:\/\/www.llama.com\/. Accessed","year":"2025","key":"e_1_3_2_1_89_1","unstructured":"Meta. [n. d.]. Llama AI Model. https:\/\/www.llama.com\/. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1260"},{"key":"e_1_3_2_1_91_1","unstructured":"Iman Mirzadeh Keivan Alizadeh Hooman Shahrokhi Oncel Tuzel Samy Bengio and Mehrdad Farajtabar. 2024. GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models. arXiv:2410.05229 [cs.LG] https:\/\/arxiv.org\/abs\/2410.05229"},{"key":"e_1_3_2_1_92_1","volume-title":"d.]. AI: A Guide for Thinking Humans. https:\/\/aiguide. substack.com. Accessed","author":"Mitchell Melanie","year":"2025","unstructured":"Melanie Mitchell. [n. d.]. AI: A Guide for Thinking Humans. https:\/\/aiguide. substack.com. Accessed: 17 Feb 2025."},{"volume-title":"Artificial Intelligence: A Guide for Thinking Humans","author":"Mitchell M.","key":"e_1_3_2_1_93_1","unstructured":"M. Mitchell. 2020. Artificial Intelligence: A Guide for Thinking Humans. Penguin Books."},{"key":"e_1_3_2_1_94_1","volume-title":"Why AI is harder than we think. arXiv preprint arXiv:2104.12871","author":"Mitchell Melanie","year":"2021","unstructured":"Melanie Mitchell. 2021. Why AI is harder than we think. arXiv preprint arXiv:2104.12871 (2021)."},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.adm8175"},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.adj5957"},{"key":"e_1_3_2_1_97_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.ado7069"},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73147-1_7"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00681"},{"key":"e_1_3_2_1_100_1","doi-asserted-by":"publisher","DOI":"10.5555\/262192.262203"},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0953-5438(98)00012-5"},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"crossref","unstructured":"Volodymyr Mnih Koray Kavukcuoglu David Silver Andrei A Rusu Joel Veness Marc G Bellemare Alex Graves Martin Riedmiller Andreas K Fidjeland Georg Ostrovski et al. 2015. Human-level control through deep reinforcement learning. nature 518 7540 (2015) 529-533.","DOI":"10.1038\/nature14236"},{"key":"e_1_3_2_1_103_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2022.3211668"},{"key":"e_1_3_2_1_104_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531924"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_1_106_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.765"},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.67"},{"key":"e_1_3_2_1_108_1","volume-title":"MTEB: Massive Text Embedding Benchmark. arXiv:2210.07316 [cs.CL] https:\/\/arxiv.org\/abs\/2210.07316","author":"Muennighoff Niklas","year":"2023","unstructured":"Niklas Muennighoff, Nouamane Tazi, Lo\u00efc Magne, and Nils Reimers. 2023. MTEB: Massive Text Embedding Benchmark. arXiv:2210.07316 [cs.CL] https:\/\/arxiv.org\/abs\/2210.07316"},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591871"},{"key":"e_1_3_2_1_110_1","volume-title":"Information, Division on Engineering, Physical Sciences, Committee on Applied, Theoretical Statistics, Board on Mathematical Sciences, et al.","author":"National Academies of Sciences, Policy, Global Affairs","year":"2019","unstructured":"National Academies of Sciences, Policy, Global Affairs, Board on Research Data, Information, Division on Engineering, Physical Sciences, Committee on Applied, Theoretical Statistics, Board on Mathematical Sciences, et al. 2019. Reproducibility and replicability in science. National Academies Press."},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.5555\/3600270.3602281"},{"key":"e_1_3_2_1_112_1","unstructured":"Xudong Pan Jiarun Dai Yihe Fan and Min Yang. 2024. Frontier AI systems have surpassed the self-replicating red line. arXiv:2412.12140 [cs.CL] https:\/\/arxiv.org\/abs\/2412.12140"},{"key":"e_1_3_2_1_113_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3474234"},{"key":"e_1_3_2_1_114_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICAIC63015.2025.10848877"},{"volume-title":"Probabilistic Reasoning in Intelligent Systems. Morgan Kaufmann","author":"Pearl Judea","key":"e_1_3_2_1_115_1","unstructured":"Judea Pearl. 1988. Probabilistic Reasoning in Intelligent Systems. Morgan Kaufmann, San Francisco, CA."},{"key":"e_1_3_2_1_116_1","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-4832-1451-1.50006-8"},{"key":"e_1_3_2_1_117_1","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-55860-332-5.50062-6"},{"key":"e_1_3_2_1_118_1","volume-title":"Yonadav Shavit, and Jonathan Frankle.","author":"Peng Andi","year":"2022","unstructured":"Andi Peng, Jessica Zosa Forde, Yonadav Shavit, and Jonathan Frankle. 2022. Strengthening subcommunities: Towards sustainable growth in ai research. arXiv preprint arXiv:2204.08377 (2022)."},{"key":"e_1_3_2_1_119_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_27"},{"key":"e_1_3_2_1_120_1","doi-asserted-by":"crossref","unstructured":"Long Phan Alice Gatti Ziwen Han Nathaniel Li Josephina Hu Hugh Zhang Sean Shi Michael Choi Anish Agrawal Arnav Chopra et al. 2025. Humanity's Last Exam. arXiv preprint arXiv:2501.14249 (2025).","DOI":"10.70777\/si.v2i1.13973"},{"key":"e_1_3_2_1_121_1","volume-title":"Chenguang Zhu, and Michael Zeng.","author":"Pryzant Reid","year":"2023","unstructured":"Reid Pryzant, Dan Iter, Jerry Li, Yin Tat Lee, Chenguang Zhu, and Michael Zeng. 2023. Automatic prompt optimization with ''gradient descent'' and beam search. arXiv preprint arXiv:2305.03495 (2023)."},{"key":"e_1_3_2_1_122_1","doi-asserted-by":"crossref","unstructured":"P Radanliev and O Santos. 2023. Adversarial attacks can deceive AI systems leading to misclassification or incorrect decisions.","DOI":"10.20944\/preprints202309.2064.v1"},{"key":"e_1_3_2_1_123_1","unstructured":"Alec Radford Rafal Jozefowicz and Ilya Sutskever. 2017. Learning to Generate Reviews and Discovering Sentiment. (2017). arXiv:1704.01444 [cs.LG] https:\/\/arxiv.org\/abs\/1704.01444"},{"key":"e_1_3_2_1_124_1","volume-title":"International conference on machine learning. PMLR, 28492-28518","author":"Radford Alec","year":"2023","unstructured":"Alec Radford, JongWook Kim, Tao Xu, Greg Brockman, Christine McLeavey, and Ilya Sutskever. 2023. Robust speech recognition via large-scale weak supervision. In International conference on machine learning. PMLR, 28492-28518."},{"key":"e_1_3_2_1_125_1","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans and Ilya Sutskever. 2018. Improving Language Understanding by Generative Pre-Training. Technical Report. OpenAI. https:\/\/cdn.openai.com\/research-covers\/language-unsupervised\/language_understanding_paper.pdf"},{"key":"e_1_3_2_1_126_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672519"},{"key":"e_1_3_2_1_127_1","volume-title":"Kochenderfer","author":"Reuel Anka","year":"2024","unstructured":"Anka Reuel, Amelia Hardy, Chandler Smith, Max Lamparth, Malcolm Hardy, and Mykel J. Kochenderfer. 2024. BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices. arXiv:2411.12990 [cs.AI] https:\/\/arxiv.org\/abs\/2411.12990"},{"key":"e_1_3_2_1_128_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Roberts Manley","year":"2023","unstructured":"Manley Roberts, Himanshu Thakur, Christine Herlihy, Colin White, and Samuel Dooley. 2023. To the cutoff... and beyond? a longitudinal perspective on LLM data contamination. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_2_1_129_1","first-page":"109","article-title":"Okapi at TREC-3","volume":"109","author":"Robertson Stephen E","year":"1995","unstructured":"Stephen E Robertson, SteveWalker, Susan Jones, MichelineMHancock-Beaulieu, Mike Gatford, et al. 1995. Okapi at TREC-3. Nist Special Publication Sp 109 (1995), 109.","journal-title":"Nist Special Publication Sp"},{"key":"e_1_3_2_1_130_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-019-09357-w"},{"key":"e_1_3_2_1_131_1","volume-title":"Rodney Brooks Vincent Conitzer","author":"Rossi Francesca","year":"2025","unstructured":"Francesca Rossi, Christian Bessiere, Joydeep Biswas, Rodney Brooks Vincent Conitzer, Thomas G. Dietterich, Virginia Dignum, Oren Etzioni, Kenneth D. Forbus, Eugene Freuder, Yolanda Gil, Holger Hoos, Eric Horvitz, Subbarao Kambhampati, Henry Kautz, Jihie Kim, Hiroaki Kitano, Alan Mackworth, Karen Myers, Luc De Raedt, Stuart Russell, Bart Selman, Peter Stone, Millind Tambe, and MichaelWooldridge. 2025. AAAI 2025 Presidential Panel on the Future of AI Research. https:\/\/aaai.org\/wp-content\/uploads\/2025\/03\/AAAI-2025-PresPanel- Report-Digital-3.7.25.pdf"},{"key":"e_1_3_2_1_132_1","volume-title":"Human Compatible: Artificial Intelligence and the Problem of Control. Allen Lane","author":"Russell Stuart","year":"2019","unstructured":"Stuart Russell. 2019. Human Compatible: Artificial Intelligence and the Problem of Control. Allen Lane, London."},{"key":"e_1_3_2_1_133_1","volume-title":"Artificial Intelligence: A Modern Approach","author":"Russell Stuart","year":"2020","unstructured":"Stuart Russell and Peter Norvig. 2020. Artificial Intelligence: A Modern Approach (4th Edition). Pearson. http:\/\/aima.cs.berkeley.edu\/","edition":"4"},{"volume-title":"Advanced topics in information retrieval, Massimo Melucci and Ricardo Beaza-Yates (Eds.)","author":"Ruthven Ian","key":"e_1_3_2_1_134_1","unstructured":"Ian Ruthven. 2011. Information retrieval in context. In Advanced topics in information retrieval, Massimo Melucci and Ricardo Beaza-Yates (Eds.). Springer, Berlin."},{"key":"e_1_3_2_1_135_1","unstructured":"Ian Ruthven and Diane Kelly (Eds.). 2011. Interactive information seeking behaviour and retrieval. Facet Publishing London."},{"key":"e_1_3_2_1_136_1","doi-asserted-by":"publisher","DOI":"10.1145\/2645710.2645746"},{"key":"e_1_3_2_1_137_1","doi-asserted-by":"publisher","DOI":"10.1145\/2792838.2792841"},{"key":"e_1_3_2_1_138_1","volume-title":"Iker Garc\u00eda-Ferrero, Julen Etxaniz, Oier Lopez de Lacalle, and Eneko Agirre.","author":"Sainz Oscar","year":"2023","unstructured":"Oscar Sainz, Jon Ander Campos, Iker Garc\u00eda-Ferrero, Julen Etxaniz, Oier Lopez de Lacalle, and Eneko Agirre. 2023. Nlp evaluation in trouble: On the need to measure llm data contamination for each benchmark. arXiv preprint arXiv:2310.18018 (2023)."},{"key":"e_1_3_2_1_139_1","doi-asserted-by":"publisher","DOI":"10.1145\/3234944.3234971"},{"key":"e_1_3_2_1_140_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451976"},{"volume-title":"Evaluating Information Retrieval and Access Tasks: NTCIR's Legacy of Research Impact","author":"Sakai Tetsuya","key":"e_1_3_2_1_141_1","unstructured":"Tetsuya Sakai, DouglasWOard, and Noriko Kando. 2021. Evaluating Information Retrieval and Access Tasks: NTCIR's Legacy of Research Impact. Springer Nature."},{"key":"e_1_3_2_1_142_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657957"},{"key":"e_1_3_2_1_143_1","doi-asserted-by":"publisher","DOI":"10.1145\/361219.361220"},{"key":"e_1_3_2_1_144_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2012.2189916"},{"key":"e_1_3_2_1_145_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_2_1_146_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_1_147_1","doi-asserted-by":"publisher","DOI":"10.1002\/ASI.4630260604"},{"key":"e_1_3_2_1_148_1","volume-title":"Critique of Honda Prize for Dr. Hinton. https: \/\/people.idsia.ch\/~juergen\/critique-honda-prize-hinton.html. Accessed","author":"Schmidhuber J\u00fcrgen","year":"2025","unstructured":"J\u00fcrgen Schmidhuber. 2020. Critique of Honda Prize for Dr. Hinton. https: \/\/people.idsia.ch\/~juergen\/critique-honda-prize-hinton.html. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_149_1","volume-title":"Proceedings on. PMLR, 103-117","author":"Schwinn Leo","year":"2023","unstructured":"Leo Schwinn, David Dobre, Stephan G\u00fcnnemann, and Gauthier Gidel. 2023. Adversarial attacks and defenses in large language models: Old and new threats. In Proceedings on. PMLR, 103-117."},{"key":"e_1_3_2_1_150_1","volume-title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting. arXiv preprint arXiv:2310.11324","author":"Sclar Melanie","year":"2023","unstructured":"Melanie Sclar, Yejin Choi, Yulia Tsvetkov, and Alane Suhr. 2023. Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting. arXiv preprint arXiv:2310.11324 (2023)."},{"key":"e_1_3_2_1_151_1","doi-asserted-by":"publisher","DOI":"10.1145\/3649468"},{"key":"e_1_3_2_1_152_1","volume-title":"Report on the 2nd Workshop on Task- Focused IR in the Era of Generative AI. SIGIR Forum 58","author":"Shah Chirag","year":"2024","unstructured":"Chirag Shah and Ryen W White. 2024. Report on the 2nd Workshop on Task- Focused IR in the Era of Generative AI. SIGIR Forum 58, 2 (2024)."},{"key":"e_1_3_2_1_153_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.118221"},{"key":"e_1_3_2_1_154_1","unstructured":"Weijia Shi Anirudh Ajith Mengzhou Xia Yangsibo Huang Daogao Liu Terra Blevins Danqi Chen and Luke Zettlemoyer. 2024. Detecting Pretraining Data from Large Language Models. arXiv:2310.16789 [cs.CL] https:\/\/arxiv.org\/abs\/ 2310.16789"},{"key":"e_1_3_2_1_155_1","doi-asserted-by":"publisher","DOI":"10.1038\/nature24270"},{"key":"e_1_3_2_1_156_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.respol.2013.05.008"},{"key":"e_1_3_2_1_157_1","volume-title":"Ilya Sutskever on Twitter\/X. https:\/\/x.com\/ilyasut\/status\/1491554478243258368. Accessed","author":"Sutskever Ilya","year":"2025","unstructured":"Ilya Sutskever. 2022. Ilya Sutskever on Twitter\/X. https:\/\/x.com\/ilyasut\/status\/1491554478243258368. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_158_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41593-023-01304-9"},{"key":"e_1_3_2_1_159_1","volume-title":"Advances in Neural Information Processing Systems","author":"Tang Qiaoyu","year":"2024","unstructured":"Qiaoyu Tang, Jiawei Chen, Zhuoqun Li, Bowen Yu, Yaojie Lu, , Haiyang Yu, Hongyu Lin, Fei Huang, Ben He, Xianpei Han, Le Sun, and Yongbin Li. 2024. Self-Retrieval: End-to-End Information Retrieval with One Large Language Model. In Advances in Neural Information Processing Systems, A. Globerson, L. Mackey, D. Belgrave, A. Fan, U. Paquet, J. Tomczak, and C. Zhang (Eds.), Vol. 37. Curran Associates, Inc., 63510-63533. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2024\/file\/741ad162ab0f3da6f9aad60e9e34f5f1-Paper-Conference.pdf"},{"key":"e_1_3_2_1_160_1","volume-title":"Gemini: A Family of Highly Capable Multimodal Models. arXiv:2312.11805 [cs.CL] https:\/\/arxiv.org\/abs\/2312.11805","author":"Team Gemini","year":"2024","unstructured":"Gemini Team. 2024. Gemini: A Family of Highly Capable Multimodal Models. arXiv:2312.11805 [cs.CL] https:\/\/arxiv.org\/abs\/2312.11805"},{"key":"e_1_3_2_1_161_1","volume-title":"REtreival Conferencer (TREC).","author":"Text","year":"1999","unstructured":"Text REtreival Conferencer (TREC). 1999. Question Answering Collection. online. https:\/\/trec.nist.gov\/data\/qa.html."},{"key":"e_1_3_2_1_162_1","volume-title":"BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. CoRR abs\/2104.08663","author":"Thakur Nandan","year":"2021","unstructured":"Nandan Thakur, Nils Reimers, Andreas R\u00fcckl\u00e9, Abhishek Srivastava, and Iryna Gurevych. 2021. BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models. CoRR abs\/2104.08663 (2021). arXiv:2104.08663 https:\/\/arxiv.org\/abs\/2104.08663"},{"key":"e_1_3_2_1_163_1","doi-asserted-by":"publisher","DOI":"10.1145\/2637002.2637032"},{"key":"e_1_3_2_1_164_1","unstructured":"University of Galsgow. [n. d.]. Terrier IR Platform. http:\/\/terrier.org\/. Accessed: 17 Feb 2025."},{"key":"e_1_3_2_1_165_1","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"Voorhees Ellen","year":"2005","unstructured":"Ellen Voorhees and Donna Harman. 2005. TREC: Experiment and evaluation in information retrieval. MIT Press."},{"volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"Voorhees Ellen M.","key":"e_1_3_2_1_166_1","unstructured":"Ellen M. Voorhees. 2002. The Philosophy of Information Retrieval Evaluation. In Evaluation of Cross-Language Information Retrieval Systems, Carol Peters, Martin Braschler, Julio Gonzalo, and Michael Kluck (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 355-370."},{"key":"e_1_3_2_1_167_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3402427"},{"key":"e_1_3_2_1_168_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564432"},{"key":"e_1_3_2_1_169_1","volume-title":"Can Old TREC Collections Reliably Evaluate Modern Neural Retrieval Models? arXiv preprint arXiv:2201.11086","author":"Voorhees Ellen M","year":"2022","unstructured":"Ellen M Voorhees, Ian Soboroff, and Jimmy Lin. 2022. Can Old TREC Collections Reliably Evaluate Modern Neural Retrieval Models? arXiv preprint arXiv:2201.11086 (2022)."},{"key":"e_1_3_2_1_170_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"},{"key":"e_1_3_2_1_171_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-4413"},{"key":"e_1_3_2_1_172_1","volume-title":"Dawn Lawrie, and Luca Soldaini.","author":"Weller Orion","year":"2024","unstructured":"Orion Weller, Benjamin Chang, Sean MacAvaney, Kyle Lo, Arman Cohan, Benjamin Van Durme, Dawn Lawrie, and Luca Soldaini. 2024. FollowIR: Evaluating and Teaching Information Retrieval Models to FollowInstructions. arXiv preprint arXiv:2403.15246 (2024)."},{"volume-title":"Interactions with search systems","author":"White Ryen W","key":"e_1_3_2_1_173_1","unstructured":"Ryen W White. 2016. Interactions with search systems. Cambridge University Press."},{"key":"e_1_3_2_1_174_1","doi-asserted-by":"publisher","DOI":"10.1007\/1-4020-3467-9_14"},{"volume-title":"Artificial intelligence","author":"Winston Patrick Henry","key":"e_1_3_2_1_175_1","unstructured":"Patrick Henry Winston. 1992. Artificial intelligence. Addison-Wesley Longman Publishing Co., Inc."},{"volume-title":"Where We Are, and Where We Are Going","author":"Wooldridge M.","key":"e_1_3_2_1_176_1","unstructured":"M. Wooldridge. 2022. A Brief History of Artificial Intelligence: What It Is, Where We Are, and Where We Are Going. Flatiron Books. https:\/\/books.google.com.au\/books?id=hjctEAAAQBAJ"},{"key":"e_1_3_2_1_177_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331340"},{"key":"e_1_3_2_1_178_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1472"},{"key":"e_1_3_2_1_179_1","volume-title":"Weaker than you think: A critical look at weakly supervised learning. arXiv preprint arXiv:2305.17442","author":"Zhu Dawei","year":"2023","unstructured":"Dawei Zhu, Xiaoyu Shen, Marius Mosbach, Andreas Stephan, and Dietrich Klakow. 2023. Weaker than you think: A critical look at weakly supervised learning. arXiv preprint arXiv:2305.17442 (2023)."},{"key":"e_1_3_2_1_180_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"},{"key":"e_1_3_2_1_181_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983723"}],"event":{"name":"ICTIR '25: International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Padua Italy","acronym":"ICTIR '25"},"container-title":["Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3731120.3744612","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:20:05Z","timestamp":1755868805000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731120.3744612"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,18]]},"references-count":181,"alternative-id":["10.1145\/3731120.3744612","10.1145\/3731120"],"URL":"https:\/\/doi.org\/10.1145\/3731120.3744612","relation":{},"subject":[],"published":{"date-parts":[[2025,7,18]]},"assertion":[{"value":"2025-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}