{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T11:47:49Z","timestamp":1774352869056,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657810","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"1200-1210","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Counterfactual Ranking Evaluation with Flexible Click Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6521-7583","authenticated-orcid":false,"given":"Alexander","family":"Buchholz","sequence":"first","affiliation":[{"name":"Amazon Music, Berlin, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-9515-5456","authenticated-orcid":false,"given":"Ben","family":"London","sequence":"additional","affiliation":[{"name":"Amazon Music, Seattle, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7568-4174","authenticated-orcid":false,"given":"Giuseppe","family":"Di Benedetto","sequence":"additional","affiliation":[{"name":"Amazon Music, Berlin, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7140-2520","authenticated-orcid":false,"given":"Jan Malte","family":"Lichtenberg","sequence":"additional","affiliation":[{"name":"Amazon Music, Berlin, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2062-7594","authenticated-orcid":false,"given":"Yannik","family":"Stein","sequence":"additional","affiliation":[{"name":"Amazon Music, Berlin, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3654-3683","authenticated-orcid":false,"given":"Thorsten","family":"Joachims","sequence":"additional","affiliation":[{"name":"Amazon Music, Ithaca, NY, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD).","author":"Agarwal A.","unstructured":"A. Agarwal, S. Basu, T. Schnabel, and T. Joachims. 2017. Effective Evaluation using Logged Bandit Feedback from Multiple Loggers. In ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331202"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313697"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3291017"},{"key":"e_1_3_2_1_5_1","volume-title":"International Conference on Machine Learning. PMLR, 984--1017","author":"Aouali Imad","year":"2023","unstructured":"Imad Aouali, Victor-Emmanuel Brunel, David Rohde, and Anna Korba. 2023. Exponential smoothing for off-policy learning. In International Conference on Machine Learning. PMLR, 984--1017."},{"key":"e_1_3_2_1_6_1","volume-title":"RecSys 2023 Workshop on Learning and Evaluating Recommendations with Impressions (LERI","author":"Benedetto Giuseppe Di","year":"2023","unstructured":"Giuseppe Di Benedetto, Alexander Buchholz, Ben London, Matej Jakimov, Yannik Stein, Jan Malte Lichtenberg, Vito Bellini, and Matteo Ruffini. 2023. Contex- tual position bias estimation using a single stochastic logging policy. In RecSys 2023 Workshop on Learning and Evaluating Recommendations with Impressions (LERI 2023). https:\/\/www.amazon.science\/publications\/contextual-position-bias- estimation-using-a-single-stochastic-logging-policy"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883033"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371844"},{"key":"e_1_3_2_1_9_1","volume-title":"Giuseppe Di Benedetto, Yannik Stein, Vito Bellini, and Matteo Ruffini.","author":"Buchholz Alexander","year":"2022","unstructured":"Alexander Buchholz, Jan Malte Lichtenberg, Giuseppe Di Benedetto, Yannik Stein, Vito Bellini, and Matteo Ruffini. 2022. Low-variance estimation in the Plackett- Luce model via quasi-Monte Carlo sampling. arXiv preprint arXiv:2205.06024 (2022)."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the learning to rank challenge. PMLR, 1--24","author":"Chapelle Olivier","year":"2011","unstructured":"Olivier Chapelle and Yi Chang. 2011. Yahoo! learning to rank challenge overview. In Proceedings of the learning to rank challenge. PMLR, 1--24."},{"key":"e_1_3_2_1_11_1","volume-title":"Click models for web search. Synthesis lectures on information concepts, retrieval, and services 7, 3","author":"Chuklin Aleksandr","year":"2015","unstructured":"Aleksandr Chuklin, Ilya Markov, and Maarten de Rijke. 2015. Click models for web search. Synthesis lectures on information concepts, retrieval, and services 7, 3 (2015), 1--115."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1214\/14-STS500"},{"key":"e_1_3_2_1_14_1","volume-title":"Doubly Robust Policy Evaluation and Learning. In International Conference on Machine Learning.","author":"Dudik M.","unstructured":"M. Dudik, J. Langford, and L. Lihong. 2011. Doubly Robust Policy Evaluation and Learning. In International Conference on Machine Learning."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390392"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331238"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 35th In- ternational Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"1456","author":"Farajtabar Mehrdad","year":"2018","unstructured":"Mehrdad Farajtabar, Yinlam Chow, and Mohammad Ghavamzadeh. 2018. More Robust Doubly Robust Off-policy Evaluation. In Proceedings of the 35th In- ternational Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 80), Jennifer Dy and Andreas Krause (Eds.). PMLR, 1447--1456. https:\/\/proceedings.mlr.press\/v80\/farajtabar18a.html"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1526709.1526712"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1498759.1498818"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-06028-6_67"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313447"},{"key":"e_1_3_2_1_22_1","volume-title":"Causal inference in statistics, social, and biomedical sciences","author":"Imbens Guido W","unstructured":"Guido W Imbens and Donald B Rubin. 2015. Causal inference in statistics, social, and biomedical sciences. Cambridge University Press."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1198\/106186008X320456"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130332.3130334"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1229179.1229181"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488560.3498380"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539813.3545119"},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the Workshop on On-line Trading of Exploration and Exploitation 2 (Proceedings of Machine Learning Research","volume":"36","author":"Li Lihong","year":"2012","unstructured":"Lihong Li, Wei Chu, John Langford, Taesup Moon, and Xuanhui Wang. 2012. An Unbiased Offline Evaluation of Contextual Bandit Algorithms with Gen- eralized Linear Models. In Proceedings of the Workshop on On-line Trading of Exploration and Exploitation 2 (Proceedings of Machine Learning Research, Vol. 26), Dorota Glowacka, Louis Dorard, and John Shawe-Taylor (Eds.). PMLR, Bellevue, Washington, USA, 19--36. https:\/\/proceedings.mlr.press\/v26\/li12a.html"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935878"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220028"},{"key":"e_1_3_2_1_32_1","volume-title":"Matteo Ruffini, and Ben London.","author":"Lichtenberg Jan Malte","year":"2023","unstructured":"Jan Malte Lichtenberg, Alexander Buchholz, Giuseppe Di Benedetto, Matteo Ruffini, and Ben London. 2023. Double Clipping: Less-Biased Variance Reduction in Off-Policy Evaluation. arXiv preprint arXiv:2309.01120 (2023)."},{"key":"e_1_3_2_1_33_1","volume-title":"RecSys 2023 Workshop on Causality, Counterfactuals & Sequential Decision-Making (CONSEQUENCES). https:\/\/www.amazon.science\/publications\/ self-normalized-off-policy-estimators-for-ranking","author":"London Ben","year":"2023","unstructured":"Ben London, Alexander Buchholz, Giuseppe Di Benedetto, Jan Malte Lichtenberg, Yannik Stein, and Thorsten Joachims. 2023. Self-normalized off-policy estimators for ranking. In RecSys 2023 Workshop on Causality, Counterfactuals & Sequential Decision-Making (CONSEQUENCES). https:\/\/www.amazon.science\/publications\/ self-normalized-off-policy-estimators-for-ranking"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539597.3570395"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462830"},{"key":"e_1_3_2_1_36_1","volume-title":"Learning-to-Rank at the Speed of Sampling: Plackett- Luce Gradient Estimation With Minimal Computational Complexity. arXiv preprint arXiv:2204.10872","author":"Oosterhuis Harrie","year":"2022","unstructured":"Harrie Oosterhuis. 2022. Learning-to-Rank at the Speed of Sampling: Plackett- Luce Gradient Estimation With Minimal Computational Complexity. arXiv preprint arXiv:2204.10872 (2022)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569453"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401102"},{"key":"e_1_3_2_1_39_1","volume-title":"Simulation and the Monte Carlo method","author":"Rubinstein Reuven Y","unstructured":"Reuven Y Rubinstein and Dirk P Kroese. 2016. Simulation and the Monte Carlo method. John Wiley & Sons."},{"key":"e_1_3_2_1_40_1","volume-title":"Modeling Position Bias Ranking for Streaming Media Services. In The Web Conference.","author":"Ruffini Matteo","year":"2022","unstructured":"Matteo Ruffini, Vito Bellini, Alexander Buchholz, Giuseppe Di Benedetto, and Yannik Stein. 2022. Modeling Position Bias Ranking for Streaming Media Services. In The Web Conference."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412262"},{"key":"e_1_3_2_1_42_1","volume-title":"Policy learning for fairness in ranking. Advances in Neural Information Processing Systems 32","author":"Singh Ashudeep","year":"2019","unstructured":"Ashudeep Singh and Thorsten Joachims. 2019. Policy learning for fairness in ranking. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_2_1_43_1","volume-title":"International Conference on Machine Learning. PMLR, 9196--9205","author":"Su Yi","year":"2020","unstructured":"Yi Su, Pavithra Srinath, and Akshay Krishnamurthy. 2020. Adaptive estimator se- lection for off-policy evaluation. In International Conference on Machine Learning. PMLR, 9196--9205."},{"key":"e_1_3_2_1_44_1","unstructured":"A. Swaminathan and T. Joachims. 2015. The Self-Normalized Estimator for Counterfactual Learning. In Neural Information Processing Systems."},{"key":"e_1_3_2_1_45_1","volume-title":"Off-policy evaluation for slate recommendation. Advances in Neural Information Processing Systems 30","author":"Swaminathan Adith","year":"2017","unstructured":"Adith Swaminathan, Akshay Krishnamurthy, Alekh Agarwal, Miro Dudik, John Langford, Damien Jose, and Imed Zitouni. 2017. Off-policy evaluation for slate recommendation. Advances in Neural Information Processing Systems 30 (2017)."},{"key":"e_1_3_2_1_46_1","volume-title":"Improved Estimator Selection for Off- Policy Evaluation. In Workshop on Reinforcement Learning Theory at the 38th International Conference on Machine Learning.","author":"Tucker George","year":"2021","unstructured":"George Tucker and Jonathan Lee. 2021. Improved Estimator Selection for Off- Policy Evaluation. In Workshop on Reinforcement Learning Theory at the 38th International Conference on Machine Learning."},{"key":"e_1_3_2_1_47_1","volume-title":"July, 2024, Washington preprint arXiv:2211","author":"Udagawa Takuma","year":"2022","unstructured":"Takuma Udagawa, Haruka Kiyohara, Yusuke Narita, Yuta Saito, and Kei Tateno. 2022. Policy-Adaptive Estimator Selection for Off-Policy Evaluation. arXiv Counterfactual Ranking Evaluation with Flexible Click Models SIGIR ?24, July, 2024, Washington preprint arXiv:2211.13904 (2022)."},{"key":"e_1_3_2_1_48_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"6476","author":"Vlassis Nikos","year":"2019","unstructured":"Nikos Vlassis, Aurelien Bibaut, Maria Dimakopoulou, and Tony Jebara. 2019. On the Design of Estimators for Bandit Off-Policy Evaluation. In Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 97), Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.). PMLR, 6468--6476. https:\/\/proceedings.mlr.press\/v97\/vlassis19a.html"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488508"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the 34th In- ternational Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"3597","author":"Wang Yu-Xiang","year":"2017","unstructured":"Yu-Xiang Wang, Alekh Agarwal, and Miroslav Dud\u00edk. 2017. Optimal and Adap- tive Off-policy Evaluation in Contextual Bandits. In Proceedings of the 34th In- ternational Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 70), Doina Precup and Yee Whye Teh (Eds.). PMLR, 3589--3597. https:\/\/proceedings.mlr.press\/v70\/wang17a.html"},{"key":"e_1_3_2_1_53_1","volume-title":"Simple statistical gradient-following algorithms for connectionist reinforcement learning. Machine learning 8, 3","author":"Williams Ronald J","year":"1992","unstructured":"Ronald J Williams. 1992. Simple statistical gradient-following algorithms for connectionist reinforcement learning. Machine learning 8, 3 (1992), 229--256."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462953"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657810","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657810","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:36:11Z","timestamp":1755840971000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657810"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":54,"alternative-id":["10.1145\/3626772.3657810","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657810","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}