{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:40:02Z","timestamp":1755870002466,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100006374","name":"Sapienza Universit\u00e0 di Roma","doi-asserted-by":"publisher","award":["2022AEF-HAZ","PE0000013","PE00000014"],"award-info":[{"award-number":["2022AEF-HAZ","PE0000013","PE00000014"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,18]]},"DOI":"10.1145\/3731120.3744575","type":"proceedings-article","created":{"date-parts":[[2025,7,18]],"date-time":"2025-07-18T13:34:06Z","timestamp":1752845646000},"page":"103-114","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["QPP-RA: Aggregating Large Language Model Rankings"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-0945-9688","authenticated-orcid":false,"given":"Filippo","family":"Betello","sequence":"first","affiliation":[{"name":"Sapienza Universit\u00e0 di Roma, Rome, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2047-4089","authenticated-orcid":false,"given":"Matteo","family":"Russo","sequence":"additional","affiliation":[{"name":"Sapienza Universit\u00e0 di Roma, Rome, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0635-6812","authenticated-orcid":false,"given":"Paul","family":"D\u00fctting","sequence":"additional","affiliation":[{"name":"Google Research, Zurich, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9809-7191","authenticated-orcid":false,"given":"Stefano","family":"Leonardi","sequence":"additional","affiliation":[{"name":"Sapienza Universit\u00e0 di Roma, Rome, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7669-9055","authenticated-orcid":false,"given":"Fabrizio","family":"Silvestri","sequence":"additional","affiliation":[{"name":"Sapienza Universit\u00e0 di Roma, Rome, Italy"}]}],"member":"320","published-online":{"date-parts":[[2025,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56069-9_51"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3610643"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2003.1196112"},{"key":"e_1_3_2_1_4_1","volume-title":"A reproducible analysis of sequential recommender systems","author":"Betello Filippo","year":"2024","unstructured":"Filippo Betello, Antonio Purificato, Federico Siciliano, Giovanni Trappolini, Andrea Bacciu, Nicola Tonellotto, and Fabrizio Silvestri. 2024. A reproducible analysis of sequential recommender systems. IEEE Access (2024)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56060-6_14"},{"key":"e_1_3_2_1_6_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020) 1877-1901."},{"key":"e_1_3_2_1_7_1","volume-title":"Principled instructions are all you need for questioning llama-1\/2, gpt-3.5\/4. arXiv preprint arXiv:2312.16171","author":"Bsharat Sondos Mahmoud","year":"2023","unstructured":"Sondos Mahmoud Bsharat, Aidar Myrzakhan, and Zhiqiang Shen. 2023. Principled instructions are all you need for questioning llama-1\/2, gpt-3.5\/4. arXiv preprint arXiv:2312.16171 (2023)."},{"volume-title":"Estimating the query difficulty for information retrieval","author":"Carmel David","key":"e_1_3_2_1_8_1","unstructured":"David Carmel and Elad Yom-Tov. 2010. Estimating the query difficulty for information retrieval. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_9_1","volume-title":"Less is More: Using Multiple LLMs for Applications with Lower Costs. (July","author":"Chen Lingjiao","year":"2023","unstructured":"Lingjiao Chen, Matei Zaharia, and James Zou. 2023. Less is More: Using Multiple LLMs for Applications with Lower Costs. (July 2023)."},{"key":"e_1_3_2_1_10_1","volume-title":"TourRank: Utilizing Large Language Models for Documents Ranking with a Tournament-Inspired Strategy. arXiv [cs.IR] (June","author":"Chen Yiqun","year":"2024","unstructured":"Yiqun Chen, Qi Liu, Yi Zhang, Weiwei Sun, Daiting Shi, Jiaxin Mao, and Dawei Yin. 2024. TourRank: Utilizing Large Language Models for Documents Ranking with a Tournament-Inspired Strategy. arXiv [cs.IR] (June 2024)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572114"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-335.deep-overview"},{"key":"e_1_3_2_1_13_1","volume-title":"Overview of the TREC 2022 Deep Learning Track. In TREC. https:\/\/trec.nist.gov\/pubs\/trec31\/papers\/Overview_deep.pdf","author":"Craswell Nick","year":"2022","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, Jimmy Lin, Ellen M. Voorhees, and Ian Soboroff. 2022. Overview of the TREC 2022 Deep Learning Track. In TREC. https:\/\/trec.nist.gov\/pubs\/trec31\/papers\/Overview_deep.pdf"},{"key":"e_1_3_2_1_14_1","volume-title":"Overview of the TREC 2019 deep learning track. arXiv preprint arXiv:2003","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M Voorhees. 2020. Overview of the TREC 2019 deep learning track. arXiv preprint arXiv:2003.07820 (2020)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463249"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2148-2153","author":"Datta Suchana","year":"2022","unstructured":"Suchana Datta, Sean MacAvaney, Debasis Ganguly, and Derek Greene. 2022. A'Pointwise-Query, Listwise-Document'based Query Performance Prediction Approach. In Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2148-2153."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/371920.372165"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00355-011-0603-9"},{"key":"e_1_3_2_1_19_1","first-page":"41","article-title":"A Spatial Approach to Predict Performance of Conversational Search Systems","author":"Faggioli Guglielmo","year":"2023","unstructured":"Guglielmo Faggioli, Nicola Ferro, Cristina Ioana Muntean, Raffaele Perego, Nicola Tonellotto, et al. 2023. A Spatial Approach to Predict Performance of Conversational Search Systems.. In IIR. 41-46.","journal-title":"IIR."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605142"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1055558.1055568"},{"key":"e_1_3_2_1_22_1","volume-title":"LLM-Ensemble: Optimal Large Language Model Ensemble Method for E-commerce Product Attribute Value Extraction. In SIGIR","author":"Fang Chenhao","year":"2024","unstructured":"Chenhao Fang, Xiaohan Li, Zezhong Fan, Jianpeng Xu, Kaushiki Nag, Evren Korpeoglu, Sushant Kumar, and Kannan Achan. 2024. LLM-Ensemble: Optimal Large Language Model Ensemble Method for E-commerce Product Attribute Value Extraction. In SIGIR 2024."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531857"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-72240-1_26"},{"key":"e_1_3_2_1_25_1","volume-title":"Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685","author":"Hu Edward J","year":"2021","unstructured":"Edward J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen. 2021. Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685 (2021)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_27_1","volume-title":"Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, et al.","author":"Jiang Albert Q","year":"2023","unstructured":"Albert Q Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, et al. 2023. Mistral 7B. arXiv preprint arXiv:2310.06825 (2023)."},{"key":"e_1_3_2_1_28_1","volume-title":"LLM-Blender: Ensembling large language models with pairwise ranking and generative fusion. arXiv [cs.CL] (June","author":"Jiang Dongfu","year":"2023","unstructured":"Dongfu Jiang, Xiang Ren, and Bill Yuchen Lin. 2023. LLM-Blender: Ensembling large language models with pairwise ranking and generative fusion. arXiv [cs.CL] (June 2023)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Enkelejda Kasneci Kathrin Se\u00dfler Stefan K\u00fcchemann Maria Bannert Daryna Dementieva Frank Fischer Urs Gasser Georg Groh Stephan G\u00fcnnemann Eyke H\u00fcllermeier et al. 2023. ChatGPT for good? On opportunities and challenges of large language models for education. Learning and individual differences 103 (2023) 102274.","DOI":"10.1016\/j.lindif.2023.102274"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.14778\/3407790.3407855"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148197"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463238"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1002\/wics.111"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Tie-Yan Liu et al. 2009. Learning to rank for information retrieval. Foundations and Trends\u00ae in Information Retrieval 3 3 (2009) 225-331.","DOI":"10.1561\/1500000016"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242638"},{"key":"e_1_3_2_1_36_1","volume-title":"Zeroshot listwise document reranking with a large language model. arXiv preprint arXiv:2305.02156","author":"Ma Xueguang","year":"2023","unstructured":"Xueguang Ma, Xinyu Zhang, Ronak Pradeep, and Jimmy Lin. 2023. Zeroshot listwise document reranking with a large language model. arXiv preprint arXiv:2305.02156 (2023)."},{"key":"e_1_3_2_1_37_1","volume-title":"Query performance prediction using relevance judgments generated by large language models. arXiv preprint arXiv:2404.01012","author":"Meng Chuan","year":"2024","unstructured":"Chuan Meng, Negar Arabzadeh, Arian Askari, Mohammad Aliannejadi, and Maarten de Rijke. 2024. Query performance prediction using relevance judgments generated by large language models. arXiv preprint arXiv:2404.01012 (2024)."},{"key":"e_1_3_2_1_38_1","volume-title":"Passage Re-ranking with BERT. arXiv preprint arXiv:1901.04085","author":"Nogueira Rodrigo","year":"2019","unstructured":"Rodrigo Nogueira and Kyunghyun Cho. 2019. Passage Re-ranking with BERT. arXiv preprint arXiv:1901.04085 (2019)."},{"key":"e_1_3_2_1_39_1","volume-title":"Document ranking with a pretrained sequence-to-sequence model. arXiv preprint arXiv:2003.06713","author":"Nogueira Rodrigo","year":"2020","unstructured":"Rodrigo Nogueira, Zhiying Jiang, and Jimmy Lin. 2020. Document ranking with a pretrained sequence-to-sequence model. arXiv preprint arXiv:2003.06713 (2020)."},{"key":"e_1_3_2_1_40_1","volume-title":"Multi-stage document ranking with BERT. arXiv preprint arXiv:1910.14424","author":"Nogueira Rodrigo","year":"2019","unstructured":"Rodrigo Nogueira, Wei Yang, Kyunghyun Cho, and Jimmy Lin. 2019. Multi-stage document ranking with BERT. arXiv preprint arXiv:1910.14424 (2019)."},{"key":"e_1_3_2_1_41_1","volume-title":"RankVicuna: Zero-Shot Listwise Document Reranking with Open-Source Large Language Models. arXiv:2309.15088","author":"Pradeep Ronak","year":"2023","unstructured":"Ronak Pradeep, Sahel Sharifymoghaddam, and Jimmy Lin. 2023. RankVicuna: Zero-Shot Listwise Document Reranking with Open-Source Large Language Models. arXiv:2309.15088 (2023)."},{"key":"e_1_3_2_1_42_1","volume-title":"RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze! arXiv:2312.02724","author":"Pradeep Ronak","year":"2023","unstructured":"Ronak Pradeep, Sahel Sharifymoghaddam, and Jimmy Lin. 2023. RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze! arXiv:2312.02724 (2023)."},{"key":"e_1_3_2_1_43_1","volume-title":"Sheaf4rec: Sheaf neural networks for graph-based recommender systems. ACM Transactions on Recommender Systems","author":"Purificato Antonio","year":"2023","unstructured":"Antonio Purificato, Giulia Cassar\u00e0, Federico Siciliano, Pietro Li\u00f2, and Fabrizio Silvestri. 2023. Sheaf4rec: Sheaf neural networks for graph-based recommender systems. ACM Transactions on Recommender Systems (2023)."},{"key":"e_1_3_2_1_44_1","volume-title":"International Workshop on Recommender Systems for Sustainability and Social Good. Springer, 111-122","author":"Purificato Antonio","year":"2024","unstructured":"Antonio Purificato and Fabrizio Silvestri. 2024. Eco-aware graph neural networks for sustainable recommendations. In International Workshop on Recommender Systems for Sustainability and Social Good. Springer, 111-122."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Zhen Qin Rolf Jagerman Kai Hui Honglei Zhuang Junru Wu Le Yan Jiaming Shen Tianqi Liu Jialu Liu Donald Metzler et al. 2023. Large language models are effective text rankers with pairwise ranking prompting. arXiv preprint arXiv:2306.17563 (2023).","DOI":"10.18653\/v1\/2024.findings-naacl.97"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609581"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2021.103865"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Stephen Robertson Hugo Zaragoza et al. 2009. The probabilistic relevance framework: BM25 and beyond. Foundations and Trends\u00ae in Information Retrieval 3 4 (2009) 333-389.","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_1_49_1","volume-title":"arXiv preprint arXiv:2010.03073","author":"dos Santos Cicero Nogueira","year":"2020","unstructured":"Cicero Nogueira dos Santos, Xiaofei Ma, Ramesh Nallapati, Zhiheng Huang, and Bing Xiang. 2020. Beyond [CLS] through ranking by generation. arXiv preprint arXiv:2010.03073 (2020)."},{"key":"e_1_3_2_1_50_1","volume-title":"European Conference on Information Retrieval. Springer, 185-197","author":"Shokouhi Milad","year":"2007","unstructured":"Milad Shokouhi. 2007. Segmentation of search engine results for effective datafusion. In European Conference on Information Retrieval. Springer, 185-197."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2017.72"},{"key":"e_1_3_2_1_52_1","volume-title":"TREC 2018 News Track Overview.. In TREC","volume":"409","author":"Soboroff Ian","year":"2018","unstructured":"Ian Soboroff, Shudong Huang, and Donna Harman. 2018. TREC 2018 News Track Overview.. In TREC, Vol. 409. 410."},{"key":"e_1_3_2_1_53_1","volume-title":"Is ChatGPT good at search? investigating large language models as re-ranking agents. arXiv preprint arXiv:2304.09542","author":"Sun Weiwei","year":"2023","unstructured":"Weiwei Sun, Lingyong Yan, Xinyu Ma, Shuaiqiang Wang, Pengjie Ren, Zhumin Chen, Dawei Yin, and Zhaochun Ren. 2023. Is ChatGPT good at search? investigating large language models as re-ranking agents. arXiv preprint arXiv:2304.09542 (2023)."},{"key":"e_1_3_2_1_54_1","volume-title":"Kabilan Elangovan, Laura Gutierrez, Ting Fang Tan, and Daniel ShuWei Ting.","author":"Thirunavukarasu Arun James","year":"2023","unstructured":"Arun James Thirunavukarasu, Darren Shu Jeng Ting, Kabilan Elangovan, Laura Gutierrez, Ting Fang Tan, and Daniel ShuWei Ting. 2023. Large language models in medicine. Nature medicine 29, 8 (2023), 1930-1940."},{"key":"e_1_3_2_1_55_1","volume-title":"Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth\u00e9e Lacroix, Baptiste Rozi\u00e8re, Naman Goyal, Eric Hambro, Faisal Azhar, et al. 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"e_1_3_2_1_56_1","first-page":"37","volume-title":"CEUR Workshop Proceedings","volume":"3366","author":"Zendel Oleg","year":"2023","unstructured":"Oleg Zendel, Binsheng Liu, J Shane Culpepper, and Falk Scholer. 2023. Entropybased query performance prediction for neural information retrieval systems. In CEUR Workshop Proceedings, Vol. 3366. CEUR-WS, 37-44."},{"key":"e_1_3_2_1_57_1","volume-title":"LLM-RankFusion: Mitigating Intrinsic Inconsistency in LLM-based Ranking. arXiv [cs.IR] (May","author":"Zeng Yifan","year":"2024","unstructured":"Yifan Zeng, Ojas Tendolkar, Raymond Baartmans, QingyunWu, Huazheng Wang, and Lizhong Chen. 2024. LLM-RankFusion: Mitigating Intrinsic Inconsistency in LLM-based Ranking. arXiv [cs.IR] (May 2024)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277835"},{"key":"e_1_3_2_1_59_1","volume-title":"Large language models for information retrieval: A survey. arXiv preprint arXiv:2308.07107","author":"Zhu Yutao","year":"2023","unstructured":"Yutao Zhu, Huaying Yuan, Shuting Wang, Jiongnan Liu, Wenhan Liu, Chenlong Deng, Zhicheng Dou, and Ji-Rong Wen. 2023. Large language models for information retrieval: A survey. arXiv preprint arXiv:2308.07107 (2023)."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3592047"}],"event":{"name":"ICTIR '25: International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Padua Italy","acronym":"ICTIR '25"},"container-title":["Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3731120.3744575","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:19:22Z","timestamp":1755868762000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731120.3744575"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,18]]},"references-count":60,"alternative-id":["10.1145\/3731120.3744575","10.1145\/3731120"],"URL":"https:\/\/doi.org\/10.1145\/3731120.3744575","relation":{},"subject":[],"published":{"date-parts":[[2025,7,18]]},"assertion":[{"value":"2025-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}