{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,27]],"date-time":"2026-03-27T17:11:25Z","timestamp":1774631485549,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,18]],"date-time":"2019-07-18T00:00:00Z","timestamp":1563408000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Ahold Delhaize"},{"name":"Association of Universities in the Netherlands"},{"name":"Innovation Center for Artificial Intelligence"},{"name":"Netherlands Organisation for Scientific Research","award":["612.001.551"],"award-info":[{"award-number":["612.001.551"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,18]]},"DOI":"10.1145\/3331184.3331269","type":"proceedings-article","created":{"date-parts":[[2019,7,19]],"date-time":"2019-07-19T13:40:26Z","timestamp":1563543626000},"page":"15-24","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":62,"title":["To Model or to Intervene"],"prefix":"10.1145","author":[{"given":"Rolf","family":"Jagerman","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Harrie","family":"Oosterhuis","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2019,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Counterfactual Learning-to-Rank for Additive Metrics and Deep Models. arXiv preprint arXiv:1805.00065","author":"Agarwal Aman","year":"2018","unstructured":"Aman Agarwal, Ivan Zaitsev, and Thorsten Joachims. 2018. Counterfactual Learning-to-Rank for Additive Metrics and Deep Models. arXiv preprint arXiv:1805.00065 (2018)."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","unstructured":"Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W. Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. In SIGIR. ACM 385--394. 10.1145\/3209978.3209986","DOI":"10.1145\/3209978.3209986"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","unstructured":"Qingyao Ai Jiaxin Mao Yiqun Liu and W. Bruce Croft. 2018. Unbiased Learning to Rank: Theory and Practice. In CIKM. ACM 2305--2306. 10.1145\/3269206.3274274","DOI":"10.1145\/3269206.3274274"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","unstructured":"Michael Bendersky Xuanhui Wang Donald Metzler and Marc Najork. 2017. Learning from User Interactions in Personal Search via Attribute Parameterization. In WSDM. ACM 791--799. 10.1145\/3018661.3018712","DOI":"10.1145\/3018661.3018712"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.5555\/3045754.3045756"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02294-4"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","unstructured":"Artem Grotov and Maarten de Rijke. 2016. Online Learning to Rank for Information Retrieval: SIGIR 2016 Tutorial. In SIGIR. ACM 1215--1218. 10.1145\/2911451.2914798","DOI":"10.1145\/2911451.2914798"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","unstructured":"Katja Hofmann Anne Schuth Shimon Whiteson and Maarten de Rijke. 2013. Reusing Historical Interaction Data for Faster Online Learning to Rank for IR. In WSDM. ACM 183--192. 10.1145\/2433396.2433419","DOI":"10.1145\/2433396.2433419"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","unstructured":"Katja Hofmann Shimon Whiteson and Maarten de Rijke. 2011. A Probabilistic Method for Inferring Preferences from Clicks. In CIKM. ACM 249--258. 10.1145\/2063576.2063618","DOI":"10.1145\/2063576.2063618"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","unstructured":"Thorsten Joachims. 2002. Optimizing Search Engines using Clickthrough Data. SIGKDD. ACM 133--142. 10.1145\/775047.775067","DOI":"10.1145\/775047.775067"},{"key":"e_1_3_2_2_12_1","unstructured":"Thorsten Joachims. 2003. Evaluating Retrieval Performance using Clickthrough Data. Text Mining. Physica\/Springer."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","unstructured":"Thorsten Joachims Laura Granka Bing Pan Helene Hembrooke and Geri Gay. 2005. Accurately Interpreting Clickthrough Data as Implicit Feedback. In SIGIR. ACM 154--161. 10.1145\/1076034.1076063","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1229179.1229181"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914803"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","unstructured":"Thorsten Joachims Adith Swaminathan and Tobias Schnabel. 2017. Unbiased Learning-to-Rank with Biased Feedback. In WSDM. ACM 781--789. 10.1145\/3018661.3018699","DOI":"10.1145\/3018661.3018699"},{"key":"e_1_3_2_2_17_1","volume-title":"BubbleRank: Safe Online Learning to Rerank. arXiv preprint arXiv:1806.05819","author":"Kveton Branislav","year":"2018","unstructured":"Branislav Kveton, Chang Li, Tor Lattimore, Ilya Markov, Maarten de Rijke, Csaba Szepesvari, and Masrour Zoghi. 2018. BubbleRank: Safe Online Learning to Rerank. arXiv preprint arXiv:1806.05819 (2018)."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","unstructured":"Lihong Li Wei Chu John Langford and Xuanhui Wang. 2011. Unbiased Offline Evaluation of Contextual-Bandit-Based News Article Recommendation Algorithms. In WSDM. ACM 297--306. 10.1145\/1935826.1935878","DOI":"10.1145\/1935826.1935878"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000016"},{"key":"e_1_3_2_2_20_1","volume-title":"Proceedings of SIGIR 2007 workshop on learning to rank for information retrieval","volume":"310","author":"Liu Tie-Yan","year":"2007","unstructured":"Tie-Yan Liu, Jun Xu, Tao Qin, Wenying Xiong, and Hang Li. 2007. Letor: Benchmark Dataset for Research on Learning to Rank for Information Retrieval. In Proceedings of SIGIR 2007 workshop on learning to rank for information retrieval, Vol. 310. ACM Amsterdam, The Netherlands."},{"key":"e_1_3_2_2_21_1","volume-title":"Learning to Rank and Evaluation in the Online Setting. 12th Russian Summer School in Information Retrieval (RuSSIR","author":"Oosterhuis Harrie","year":"2018","unstructured":"Harrie Oosterhuis. 2018. Learning to Rank and Evaluation in the Online Setting. 12th Russian Summer School in Information Retrieval (RuSSIR 2018). (2018)."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2017. Balancing Speed and Quality in Online Learning to Rank for Information Retrieval. In CIKM. ACM 277--286. 10.1145\/3132847.3132896","DOI":"10.1145\/3132847.3132896"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2017. Sensitive and Scalable Online Evaluation with Theoretical Guarantees. In CIKM. ACM 77--86. 10.1145\/3132847.3132895","DOI":"10.1145\/3132847.3132895"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2018. Differentiable Unbiased Online Learning to Rank. CIKM. ACM 1293--1302. 10.1145\/3269206.3271686","DOI":"10.1145\/3269206.3271686"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-15712-8_25"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_50"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","unstructured":"Filip Radlinski and Nick Craswell. 2013. Optimized Interleaving for Online Retrieval Evaluation. In WSDM. ACM 245--254. 10.1145\/2433396.2433429","DOI":"10.1145\/2433396.2433429"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","unstructured":"Anne Schuth Harrie Oosterhuis Shimon Whiteson and Maarten de Rijke. 2016. Multileave Gradient Descent for Fast Online Learning to Rank. In WSDM. ACM 457--466. 10.1145\/2835776.2835804","DOI":"10.1145\/2835776.2835804"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","unstructured":"Anne Schuth Floor Sietsma Shimon Whiteson Damien Lefortier and Maarten de Rijke. 2014. Multileaved Comparisons for Fast Online Evaluation. In CIKM. ACM 71--80. 10.1145\/2661829.2661952","DOI":"10.1145\/2661829.2661952"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.5555\/3019233"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","unstructured":"Adith Swaminathan and Thorsten Joachims. 2015. Counterfactual Risk Minimization: Learning from Logged Bandit Feedback. In ICML. PMLR 814--823.","DOI":"10.5555\/3045118.3045206"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","unstructured":"Adith Swaminathan Akshay Krishnamurthy Alekh Agarwal Miro Dudik John Langford Damien Jose and Imed Zitouni. 2017. Off-policy Evaluation for Slate Recommendation. In NIPS. 3632--3642.","DOI":"10.5555\/3294996.3295121"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2015.07.002"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","unstructured":"Philip S. Thomas Georgios Theocharous and Mohammad Ghavamzadeh. 2015. High-Confidence Off-Policy Evaluation. In AAAI. 3000--3006.","DOI":"10.5555\/2888116.2888134"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","unstructured":"Xuanhui Wang Michael Bendersky Donald Metzler and Marc Najork. 2016. Learning to Rank with Selection Bias in Personal Search. In SIGIR. ACM 115--124. 10.1145\/2911451.2911537","DOI":"10.1145\/2911451.2911537"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","unstructured":"Yisong Yue and Thorsten Joachims. 2009. Interactively Optimizing Information Retrieval Systems as a Dueling Bandits Problem. In ICML. ACM 1201--1208. 10.1145\/1553374.1553527","DOI":"10.1145\/1553374.1553527"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","unstructured":"Tong Zhao and Irwin King. 2016. Constructing Reliable Gradient Exploration for Online Learning to Rank. In CIKM. ACM 1643--1652. 10.1145\/2983323.2983774","DOI":"10.1145\/2983323.2983774"}],"event":{"name":"SIGIR '19: The 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Paris France","acronym":"SIGIR '19","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331269","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3331184.3331269","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T00:53:07Z","timestamp":1768524787000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331269"}},"subtitle":["A Comparison of Counterfactual and Online Learning to Rank from User Interactions"],"short-title":[],"issued":{"date-parts":[[2019,7,18]]},"references-count":38,"alternative-id":["10.1145\/3331184.3331269","10.1145\/3331184"],"URL":"https:\/\/doi.org\/10.1145\/3331184.3331269","relation":{},"subject":[],"published":{"date-parts":[[2019,7,18]]},"assertion":[{"value":"2019-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}