{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T15:34:06Z","timestamp":1774366446981,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":74,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,23]],"date-time":"2022-08-23T00:00:00Z","timestamp":1661212800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Google Inc.","award":["Google Research Scholar Program 2021"],"award-info":[{"award-number":["Google Research Scholar Program 2021"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,23]]},"DOI":"10.1145\/3539813.3545137","type":"proceedings-article","created":{"date-parts":[[2022,8,25]],"date-time":"2022-08-25T22:18:32Z","timestamp":1661465912000},"page":"264-274","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":18,"title":["Reaching the End of Unbiasedness"],"prefix":"10.1145","author":[{"given":"Harrie","family":"Oosterhuis","sequence":"first","affiliation":[{"name":"Radboud University, Nijmegen, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2022,8,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331202"},{"key":"e_1_3_2_1_2_1","volume-title":"Addressing Trust Bias for Unbiased Learning-to-Rank. In The World Wide Web Conference. ACM, 4--14","author":"Agarwal Aman","year":"2019","unstructured":"Aman Agarwal , Xuanhui Wang , Cheng Li , Michael Bendersky , and Marc Najork . 2019 . Addressing Trust Bias for Unbiased Learning-to-Rank. In The World Wide Web Conference. ACM, 4--14 . Aman Agarwal, Xuanhui Wang, Cheng Li, Michael Bendersky, and Marc Najork. 2019. Addressing Trust Bias for Unbiased Learning-to-Rank. In The World Wide Web Conference. ACM, 4--14."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3291017"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148175"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3209986"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3439861"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210063"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911504"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883033"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210004"},{"key":"e_1_3_2_1_12_1","first-page":"1","article-title":"Yahoo! Learning to Rank Challenge Overview","volume":"14","author":"Chapelle Olivier","year":"2011","unstructured":"Olivier Chapelle and Yi Chang . 2011 . Yahoo! Learning to Rank Challenge Overview . Journal of Machine Learning Research 14 (2011), 1 -- 24 . Olivier Chapelle and Yi Chang. 2011. Yahoo! Learning to Rank Challenge Overview. Journal of Machine Learning Research 14 (2011), 1--24.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1526709.1526711"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371819"},{"key":"e_1_3_2_1_15_1","volume-title":"Click Models for Web Search","author":"Chuklin Aleksandr","unstructured":"Aleksandr Chuklin , Ilya Markov , and Maarten de Rijke . 2015. Click Models for Web Search . Morgan & Claypool Publishers . Aleksandr Chuklin, Ilya Markov, and Maarten de Rijke. 2015. Click Models for Web Search. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390446"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2959100.2959190"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348296"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2987380"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411962"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331238"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2843948"},{"key":"e_1_3_2_1_24_1","volume-title":"Sample Selection Bias as a Specification Error. Econometrica: Journal of the econometric society","author":"Heckman James J","year":"1979","unstructured":"James J Heckman . 1979. Sample Selection Bias as a Specification Error. Econometrica: Journal of the econometric society ( 1979 ), 153--161. James J Heckman. 1979. Sample Selection Bias as a Specification Error. Econometrica: Journal of the econometric society (1979), 153--161."},{"key":"e_1_3_2_1_25_1","volume-title":"Fast and Reliably Online Learning to Rank for Information Retrieval. Ph. D. Dissertation","author":"Hofmann Katja","unstructured":"Katja Hofmann . 2013. Fast and Reliably Online Learning to Rank for Information Retrieval. Ph. D. Dissertation . University of Amsterdam. Katja Hofmann. 2013. Fast and Reliably Online Learning to Rank for Information Retrieval. Ph. D. Dissertation. University of Amsterdam."},{"key":"e_1_3_2_1_26_1","volume-title":"Effects of Position Bias on Click-based Recommender Evaluation. In European Conference on Information Retrieval. Springer, 624--630","author":"Hofmann Katja","year":"2014","unstructured":"Katja Hofmann , Anne Schuth , Alejandro Bellogin , and Maarten de Rijke . 2014 . Effects of Position Bias on Click-based Recommender Evaluation. In European Conference on Information Retrieval. Springer, 624--630 . Katja Hofmann, Anne Schuth, Alejandro Bellogin, and Maarten de Rijke. 2014. Effects of Position Bias on Click-based Recommender Evaluation. In European Conference on Information Retrieval. Springer, 624--630."},{"key":"e_1_3_2_1_27_1","volume-title":"Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm. In The World Wide Web Conference. ACM, 2830--2836","author":"Hu Ziniu","year":"2019","unstructured":"Ziniu Hu , Yang Wang , Qu Peng , and Hang Li . 2019 . Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm. In The World Wide Web Conference. ACM, 2830--2836 . Ziniu Hu, Yang Wang, Qu Peng, and Hang Li. 2019. Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm. In The World Wide Web Conference. ACM, 2830--2836."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1944339.1944341"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331269"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/775047.775067"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130332.3130334"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 32nd International Conference on International Conference on Machine Learning -","volume":"37","author":"Komiyama Junpei","year":"2015","unstructured":"Junpei Komiyama , Junya Honda , and Hiroshi Nakagawa . 2015 . Optimal Regret Analysis of Thompson Sampling in Stochastic Multi-armed Bandit Problem with Multiple Plays . In Proceedings of the 32nd International Conference on International Conference on Machine Learning - Volume 37 (Lille, France) (ICML'15). JMLR.org, 1152--1161. Junpei Komiyama, Junya Honda, and Hiroshi Nakagawa. 2015. Optimal Regret Analysis of Thompson Sampling in Stochastic Multi-armed Bandit Problem with Multiple Plays. In Proceedings of the 32nd International Conference on International Conference on Machine Learning - Volume 37 (Lille, France) (ICML'15). JMLR.org, 1152--1161."},{"key":"e_1_3_2_1_35_1","unstructured":"Paul Lagr\u00e9e Claire Vernade and Olivier Capp\u00e9. 2016. Multiple-Play Bandits in the Position-Based Model. In Advances in Neural Information Processing Systems. 1597--1605.  Paul Lagr\u00e9e Claire Vernade and Olivier Capp\u00e9. 2016. Multiple-Play Bandits in the Position-Based Model. In Advances in Neural Information Processing Systems. 1597--1605."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000016"},{"key":"e_1_3_2_1_37_1","volume-title":"Individual Choice Behavior: A Theoretical Analysis","author":"Luce R Duncan","unstructured":"R Duncan Luce . 2012. Individual Choice Behavior: A Theoretical Analysis . Courier Corporation . R Duncan Luce. 2012. Individual Choice Behavior: A Theoretical Analysis. Courier Corporation."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3052768"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Marco Morik Ashudeep Singh Jessica Hong and Thorsten Joachims. 2020. Controlling Fairness and Bias in Dynamic Learning-to-Rank. ACM 429--438.  Marco Morik Ashudeep Singh Jessica Hong and Thorsten Joachims. 2020. Controlling Fairness and Bias in Dynamic Learning-to-Rank. ACM 429--438.","DOI":"10.1145\/3397271.3401100"},{"key":"e_1_3_2_1_41_1","volume-title":"Learning From User Interactions With Rankings: A Unification of the Field. Ph. D. Dissertation","author":"Oosterhuis Harrie","unstructured":"Harrie Oosterhuis . 2020. Learning From User Interactions With Rankings: A Unification of the Field. Ph. D. Dissertation . University of Amsterdam. Harrie Oosterhuis. 2020. Learning From User Interactions With Rankings: A Unification of the Field. Ph. D. Dissertation. University of Amsterdam."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462830"},{"key":"e_1_3_2_1_43_1","volume-title":"Advances in Information Retrieval","author":"Oosterhuis Harrie","unstructured":"Harrie Oosterhuis and Maarten de Rijke . 2019. Optimizing Ranking Models in an Online Setting . In Advances in Information Retrieval . Springer International Publishing , Cham , 382--396. Harrie Oosterhuis and Maarten de Rijke. 2019. Optimizing Ranking Models in an Online Setting. In Advances in Information Retrieval. Springer International Publishing, Cham, 382--396."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401102"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409820"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441794"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366424.3383107"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380255"},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1763--1767","author":"Ovaisi Zohreh","year":"2021","unstructured":"Zohreh Ovaisi , Kathryn Vasilaky , and Elena Zheleva . 2021 . PropensityIndependent Bias Recovery in Offline Learning-to-Rank Systems . In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1763--1767 . Zohreh Ovaisi, Kathryn Vasilaky, and Elena Zheleva. 2021. PropensityIndependent Bias Recovery in Offline Learning-to-Rank Systems. In Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1763--1767."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1083-6101.2007.00351.x"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.2307\/2346567"},{"key":"e_1_3_2_1_52_1","volume-title":"arXiv preprint arXiv:1306.2597","author":"Qin Tao","year":"2013","unstructured":"Tao Qin and Tie-Yan Liu . 2013. Introducing LETOR 4.0 Datasets . arXiv preprint arXiv:1306.2597 ( 2013 ). Tao Qin and Tie-Yan Liu. 2013. Introducing LETOR 4.0 Datasets. arXiv preprint arXiv:1306.2597 (2013)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148320"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Filip Radlinski Madhu Kurup and Thorsten Joachims. 2008. How Does Clickthrough Data Reflect Retrieval Quality?. In CIKM. ACM 43--52.  Filip Radlinski Madhu Kurup and Thorsten Joachims. 2008. How Does Clickthrough Data Reflect Retrieval Quality?. In CIKM. ACM 43--52.","DOI":"10.1145\/1458082.1458092"},{"key":"e_1_3_2_1_55_1","volume-title":"International Conference on Machine Learning. 837--845","author":"Raman Karthik","year":"2013","unstructured":"Karthik Raman , Thorsten Joachims , Pannaga Shivaswamy , and Tobias Schnabel . 2013 . Stable Coactive Learning via Perturbation . In International Conference on Machine Learning. 837--845 . Karthik Raman, Thorsten Joachims, Pannaga Shivaswamy, and Tobias Schnabel. 2013. Stable Coactive Learning via Perturbation. In International Conference on Machine Learning. 837--845."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000040"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409812"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Harrie Oosterhuis Shimon Whiteson and Maarten de Rijke. 2016. Multileave Gradient Descent for Fast Online Learning to Rank. In WSDM. ACM 457--466.  Anne Schuth Harrie Oosterhuis Shimon Whiteson and Maarten de Rijke. 2016. Multileave Gradient Descent for Fast Online Learning to Rank. In WSDM. ACM 457--466.","DOI":"10.1145\/2835776.2835804"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926725"},{"key":"e_1_3_2_1_62_1","volume-title":"Culotta (Eds.)","volume":"23","author":"Strehl Alex","year":"2010","unstructured":"Alex Strehl , John Langford , Lihong Li , and Sham M Kakade . 2010 . Learning from Logged Implicit Exploration Data. In Advances in Neural Information Processing Systems, J. Lafferty, C. Williams, J. Shawe-Taylor, R. Zemel, and A . Culotta (Eds.) , Vol. 23 . Curran Associates, Inc. Alex Strehl, John Langford, Lihong Li, and Sham M Kakade. 2010. Learning from Logged Implicit Exploration Data. In Advances in Neural Information Processing Systems, J. Lafferty, C. Williams, J. Shawe-Taylor, R. Zemel, and A. Culotta (Eds.), Vol. 23. Curran Associates, Inc."},{"key":"e_1_3_2_1_63_1","unstructured":"Adith Swaminathan and Thorsten Joachims. 2015. The Self-Normalized Estimator for Counterfactual Learning. In Advances in Neural Information Processing Systems. 3231--3239.  Adith Swaminathan and Thorsten Joachims. 2015. The Self-Normalized Estimator for Counterfactual Learning. In Advances in Neural Information Processing Systems. 3231--3239."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401299"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482275"},{"key":"e_1_3_2_1_66_1","volume-title":"Proceedings of the 28th ACM International Conference on Information and Knowledge Management.","author":"Vardasbi Ali","year":"2020","unstructured":"Ali Vardasbi , Harrie Oosterhuis , and Maarten de Rijke . 2020 . When Inverse Propensity Scoring does not Work: Affine Corrections for Unbiased Learning to Rank . In Proceedings of the 28th ACM International Conference on Information and Knowledge Management. Ali Vardasbi, Harrie Oosterhuis, and Maarten de Rijke. 2020. When Inverse Propensity Scoring does not Work: Affine Corrections for Unbiased Learning to Rank. In Proceedings of the 28th ACM International Conference on Information and Knowledge Management."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441798"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271784"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441751"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390156.1390306"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080775"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553527"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450098"}],"event":{"name":"ICTIR '22: The 2022 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Madrid Spain","acronym":"ICTIR '22","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2022 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539813.3545137","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539813.3545137","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:03Z","timestamp":1750183803000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539813.3545137"}},"subtitle":["Uncovering Implicit Limitations of Click-Based Learning to Rank"],"short-title":[],"issued":{"date-parts":[[2022,8,23]]},"references-count":74,"alternative-id":["10.1145\/3539813.3545137","10.1145\/3539813"],"URL":"https:\/\/doi.org\/10.1145\/3539813.3545137","relation":{},"subject":[],"published":{"date-parts":[[2022,8,23]]},"assertion":[{"value":"2022-08-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}