{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T15:24:35Z","timestamp":1768577075028,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,18]],"date-time":"2019-07-18T00:00:00Z","timestamp":1563408000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,18]]},"DOI":"10.1145\/3331184.3331378","type":"proceedings-article","created":{"date-parts":[[2019,7,19]],"date-time":"2019-07-19T13:40:26Z","timestamp":1563543626000},"page":"1399-1400","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Effective Online Evaluation for Web Search"],"prefix":"10.1145","author":[{"given":"Alexey","family":"Drutsa","sequence":"first","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gleb","family":"Gusev","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eugene","family":"Kharitonov","sequence":"additional","affiliation":[{"name":"Facebook AI Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Denis","family":"Kulemyakin","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pavel","family":"Serdyukov","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Igor","family":"Yashkov","sequence":"additional","affiliation":[{"name":"Yandex, Moscow, Russian Fed."}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2019,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3041021.3054196"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767786"},{"key":"e_1_3_2_1_3_1","volume-title":"Machine Learning Methods for Estimating Heterogeneous Causal Effects. arXiv preprint arXiv:1504.01132","author":"Athey Susan","year":"2015","unstructured":"Susan Athey and Guido Imbens. 2015. Machine Learning Methods for Estimating Heterogeneous Causal Effects. arXiv preprint arXiv:1504.01132 (2015)."},{"key":"e_1_3_2_1_4_1","unstructured":"Juliette Aurisset Michael Ramm and Joshua Parks. 2017. Innovating Faster on Personalization Algorithms at Netflix Using Interleaving. https:\/\/medium.com\/netflix-techblog\/interleaving-in-online-experiments-at-netflix-a04ee392ec55."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488218"},{"key":"e_1_3_2_1_6_1","unstructured":"Roman Budylin Alexey Drutsa Gleb Gusev Eugene Kharitonov Pavel Serdyukov and Igor Yashkov. 2018a. Online Evaluation for Effective Web Service Development: Extended Abstract of the Tutorial at TheWebConf'2018."},{"key":"e_1_3_2_1_7_1","unstructured":"Roman Budylin Alexey Drutsa Gleb Gusev Pavel Serdyukov and Igor Yashkov. 2018b. Online evaluation for effective web service development. In arXiv preprint arXiv:1809.00661. Tutorial at KDD'2018."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159699"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609535"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940716.2940757"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2882984"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557139"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2742563"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685307"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2568028"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"},{"key":"e_1_3_2_1_20_1","unstructured":"Pavel Dmitriev Somit Gupta Ron Kohavi Alex Deng Paul Raff and Lukas Vermeer. 2017. A\/B Testing at Scale. https:\/\/exp-platform.com\/2017abtestingtutorial\/"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983356"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767814"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685318"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741116"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2856822"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052664"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806496"},{"key":"e_1_3_2_1_28_1","volume-title":"An introduction to the bootstrap","author":"Efron Bradley","unstructured":"Bradley Efron and Robert J Tibshirani. 1994. An introduction to the bootstrap .CRC press."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","unstructured":"Artem Grotov and Maarten de Rijke. 2016. Online learning to rank for information retrieval: Tutorial. In SIGIR. 10.1145\/2911451.2914798","DOI":"10.1145\/2911451.2914798"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741081"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063618"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788583"},{"key":"e_1_3_2_1_33_1","unstructured":"Thorsten Joachims. 2002. Unbiased evaluation of retrieval quality using clickthrough data. (2002)."},{"key":"e_1_3_2_1_34_1","unstructured":"Thorsten Joachims et al. 2003. Evaluating Retrieval Performance Using Clickthrough Data."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3097992"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","unstructured":"Rosie Jones and Kristina Lisa Klinkner. 2008. Beyond the session timeout: automatic hierarchical segmentation of search topics in query logs. In CIKM. 699--708. 10.1145\/1458082.1458176","DOI":"10.1145\/1458082.1458176"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018708"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806477"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767706"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767729"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556220"},{"key":"e_1_3_2_1_42_1","volume-title":"Juan Lavista Ferres, and Tamir Melamed","author":"Kohavi Ronny","year":"2009","unstructured":"Ronny Kohavi, Thomas Crook, Roger Longbotham, Brian Frasca, Randy Henne, Juan Lavista Ferres, and Tamir Melamed. 2009a. Online experimentation at Microsoft. Data Mining Case Studies (2009), 11."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","unstructured":"R. Kohavi A. Deng R. Longbotham and Y. Xu. 2014. Seven Rules of Thumb for Web Site Experimenters. In KDD'2014. 10.1145\/2623330.2623341","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281295"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_48_1","volume-title":"Randy Henne, Vignesh Kannappan, and Justin Wang.","author":"Kohavi Ron","year":"2010","unstructured":"Ron Kohavi, David Messner, Seth Eliot, Juan Lavista Ferres, Randy Henne, Vignesh Kannappan, and Justin Wang. 2010. Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.5555\/2787876"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308774.3308780"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2783415"},{"key":"e_1_3_2_1_52_1","unstructured":"Eric T Peterson. 2004. Web analytics demystified: a marketer's guide to understanding how your web site affects your business .Ingram."},{"key":"e_1_3_2_1_53_1","volume-title":"Some methods for heterogeneous treatment effect estimation in high-dimensions. arXiv preprint arXiv:1707.00102","author":"Powers Scott","year":"2017","unstructured":"Scott Powers, Junyang Qian, Kenneth Jung, Alejandro Schuler, Nigam H Shah, Trevor Hastie, and Robert Tibshirani. 2017. Some methods for heterogeneous treatment effect estimation in high-dimensions. arXiv preprint arXiv:1707.00102 (2017)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939688"},{"key":"e_1_3_2_1_55_1","unstructured":"Filip Radlinski. 2013. Sensitive Online Search Evaluation. http:\/\/irsg.bcs.org\/SearchSolutions\/2013\/presentations\/radlinski.pdf."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","unstructured":"Filip Radlinski and Nick Craswell. 2013. Optimized interleaving for online retrieval evaluation. In WSDM. 10.1145\/2433396.2433429","DOI":"10.1145\/2433396.2433429"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","unstructured":"Filip Radlinski and Katja Hofmann. 2013. Practical online retrieval evaluation. In ECIR. 10.1007\/978-3-642-36973-5_107","DOI":"10.1007\/978-3-642-36973-5_107"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458092"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","unstructured":"Filip Radlinski and Yisong Yue. 2011. Practical Online Retrieval Evaluation. In SIGIR. 10.1145\/2009916.2010171","DOI":"10.1145\/2009916.2010171"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753687"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098192"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","unstructured":"Anne Schuth Floor Sietsma Shimon Whiteson Damien Lefortier and Maarten de Rijke. 2014. Multileaved comparisons for fast online evaluation. In CIKM. 10.1145\/2661829.2661952","DOI":"10.1145\/2661829.2661952"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010104"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488494"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","unstructured":"Diane Tang Ashish Agarwal Deirdre O'Brien and Mike Meyer. 2010. Overlapping experiment infrastructure: More better faster experimentation. In KDD. 10.1145\/1835804.1835810","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939733"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939703"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788602"}],"event":{"name":"SIGIR '19: The 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Paris France","acronym":"SIGIR '19","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331378","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3331184.3331378","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T00:55:09Z","timestamp":1768524909000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331378"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,7,18]]},"references-count":68,"alternative-id":["10.1145\/3331184.3331378","10.1145\/3331184"],"URL":"https:\/\/doi.org\/10.1145\/3331184.3331378","relation":{},"subject":[],"published":{"date-parts":[[2019,7,18]]},"assertion":[{"value":"2019-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}