{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,24]],"date-time":"2025-08-24T01:18:03Z","timestamp":1755998283759,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":24,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,9,14]],"date-time":"2020-09-14T00:00:00Z","timestamp":1600041600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100003246","name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","doi-asserted-by":"publisher","award":["612.001.551"],"award-info":[{"award-number":["612.001.551"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,9,14]]},"DOI":"10.1145\/3409256.3409820","type":"proceedings-article","created":{"date-parts":[[2020,9,5]],"date-time":"2020-09-05T10:13:32Z","timestamp":1599300812000},"page":"137-144","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["Taking the Counterfactual Online: Efficient and Unbiased Online Evaluation for Ranking"],"prefix":"10.1145","author":[{"given":"Harrie","family":"Oosterhuis","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam & Ahold Delhaize, Amsterdam, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2020,9,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Aman Agarwal Ivan Zaitsev Xuanhui Wang Cheng Li Marc Najork and Thorsten Joachims. 2019. Estimating Position Bias without Intrusive Interventions. In WSDM. ACM 474--482.  Aman Agarwal Ivan Zaitsev Xuanhui Wang Cheng Li Marc Najork and Thorsten Joachims. 2019. Estimating Position Bias without Intrusive Interventions. In WSDM. ACM 474--482.","DOI":"10.1145\/3289600.3291017"},{"key":"e_1_3_2_1_2_1","unstructured":"Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. In SIGIR. ACM 385--394.  Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. In SIGIR. ACM 385--394."},{"key":"e_1_3_2_1_3_1","first-page":"1","article-title":"Yahoo! Learning to Rank Challenge Overview","volume":"14","author":"Chapelle Olivier","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Aleksandr Chuklin Ilya Markov and Maarten de Rijke. 2015. Click Models for Web Search .Morgan & Claypool Publishers.  Aleksandr Chuklin Ilya Markov and Maarten de Rijke. 2015. Click Models for Web Search .Morgan & Claypool Publishers.","DOI":"10.1007\/978-3-031-02294-4"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Nick Craswell Onno Zoeter Michael Taylor and Bill Ramsey. 2008. An Experimental Comparison of Click Position-bias Models. In WSDM. 87--94.  Nick Craswell Onno Zoeter Michael Taylor and Bill Ramsey. 2008. An Experimental Comparison of Click Position-bias Models. In WSDM. 87--94.","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Zhichong Fang Aman Agarwal and Thorsten Joachims. 2019. Intervention Harvesting for Context-dependent Examination-bias Estimation. In SIGIR. 825--834.  Zhichong Fang Aman Agarwal and Thorsten Joachims. 2019. Intervention Harvesting for Context-dependent Examination-bias Estimation. In SIGIR. 825--834.","DOI":"10.1145\/3331184.3331238"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Katja Hofmann Shimon Whiteson and Maarten de Rijke. 2011. A Probabilistic Method for Inferring Preferences from Clicks. In CIKM. ACM 249--258.  Katja Hofmann Shimon Whiteson and Maarten de Rijke. 2011. A Probabilistic Method for Inferring Preferences from Clicks. In CIKM. ACM 249--258.","DOI":"10.1145\/2063576.2063618"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2536736.2536737"},{"key":"e_1_3_2_1_11_1","unstructured":"Thorsten Joachims. 2003. Evaluating Retrieval Performance Using Clickthrough Data. In Text Mining J. Franke G. Nakhaeizadeh and I. Renz (Eds.). Physica Verlag.  Thorsten Joachims. 2003. Evaluating Retrieval Performance Using Clickthrough Data. In Text Mining J. Franke G. Nakhaeizadeh and I. Renz (Eds.). Physica Verlag."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Thorsten Joachims Laura Granka Bing Pan Helene Hembrooke and Geri Gay. 2005. Accurately Interpreting Clickthrough Data as Implicit Feedback. In SIGIR. ACM 154--161.  Thorsten Joachims Laura Granka Bing Pan Helene Hembrooke and Geri Gay. 2005. Accurately Interpreting Clickthrough Data as Implicit Feedback. In SIGIR. ACM 154--161.","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Thorsten Joachims Adith Swaminathan and Tobias Schnabel. 2017. Unbiased Learning-to-Rank with Biased Feedback. In WSDM. ACM 781--789.  Thorsten Joachims Adith Swaminathan and Tobias Schnabel. 2017. Unbiased Learning-to-Rank with Biased Feedback. In WSDM. ACM 781--789.","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7687-1_891"},{"volume-title":"Off-policy Learning in Two-stage Recommender Systems. In The Web Conference","year":"2020","author":"Ma Jiaqi","key":"e_1_3_2_1_15_1"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2020. Policy-Aware Unbiased Learning to Rank for Top-k Rankings. In SIGIR. ACM.  Harrie Oosterhuis and Maarten de Rijke. 2020. Policy-Aware Unbiased Learning to Rank for Top-k Rankings. In SIGIR. ACM.","DOI":"10.1145\/3397271.3401102"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Zohreh Ovaisi Ragib Ahsan Yifan Zhang Kathryn Vasilaky and Elena Zheleva. 2020. Correcting for Selection Bias in Learning-to-rank Systems. In WWW. 1863--1873.  Zohreh Ovaisi Ragib Ahsan Yifan Zhang Kathryn Vasilaky and Elena Zheleva. 2020. Correcting for Selection Bias in Learning-to-rank Systems. In WWW. 1863--1873.","DOI":"10.1145\/3366423.3380255"},{"key":"e_1_3_2_1_18_1","unstructured":"Tao Qin and Tie-Yan Liu. 2013. Introducing LETOR 4.0 datasets. arXiv preprint arXiv:1306.2597 (2013).  Tao Qin and Tie-Yan Liu. 2013. Introducing LETOR 4.0 datasets. arXiv preprint arXiv:1306.2597 (2013)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Filip Radlinski and Nick Craswell. 2013. Optimized Interleaving for Online Retrieval Evaluation. In WSDM. ACM 245--254.  Filip Radlinski and Nick Craswell. 2013. Optimized Interleaving for Online Retrieval Evaluation. In WSDM. ACM 245--254.","DOI":"10.1145\/2433396.2433429"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Filip Radlinski Madhu Kurup and Thorsten Joachims. 2008. How Does Clickthrough Data Reflect Retrieval Quality?. In CIKM. ACM 43--52.  Filip Radlinski Madhu Kurup and Thorsten Joachims. 2008. How Does Clickthrough Data Reflect Retrieval Quality?. In CIKM. ACM 43--52.","DOI":"10.1145\/1458082.1458092"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting Search Satisfaction Metrics with Interleaved Comparisons. In SIGIR. 463--472.  Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting Search Satisfaction Metrics with Interleaved Comparisons. In SIGIR. 463--472.","DOI":"10.1145\/2766462.2767695"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Michael Bendersky Donald Metzler and Marc Najork. 2016. Learning to Rank with Selection Bias in Personal Search. In SIGIR. ACM 115--124.  Xuanhui Wang Michael Bendersky Donald Metzler and Marc Najork. 2016. Learning to Rank with Selection Bias in Personal Search. In SIGIR. ACM 115--124.","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Nadav Golbandi Michael Bendersky Donald Metzler and Marc Najork. 2018a. Position Bias Estimation for Unbiased Learning to Rank in Personal Search. In WSDM. ACM 610--618.  Xuanhui Wang Nadav Golbandi Michael Bendersky Donald Metzler and Marc Najork. 2018a. Position Bias Estimation for Unbiased Learning to Rank in Personal Search. In WSDM. ACM 610--618.","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Xuanhui Wang Cheng Li Nadav Golbandi Michael Bendersky and Marc Najork. 2018b. The LambdaLoss Framework for Ranking Metric Optimization. In CIKM. ACM 1313--1322.  Xuanhui Wang Cheng Li Nadav Golbandi Michael Bendersky and Marc Najork. 2018b. The LambdaLoss Framework for Ranking Metric Optimization. In CIKM. ACM 1313--1322.","DOI":"10.1145\/3269206.3271784"}],"event":{"name":"ICTIR '20: The 2020 ACM SIGIR International Conference on the Theory of Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Virtual Event Norway","acronym":"ICTIR '20"},"container-title":["Proceedings of the 2020 ACM SIGIR on International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3409256.3409820","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3409256.3409820","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:32Z","timestamp":1750197692000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3409256.3409820"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,9,14]]},"references-count":24,"alternative-id":["10.1145\/3409256.3409820","10.1145\/3409256"],"URL":"https:\/\/doi.org\/10.1145\/3409256.3409820","relation":{},"subject":[],"published":{"date-parts":[[2020,9,14]]},"assertion":[{"value":"2020-09-14","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}