{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T19:45:09Z","timestamp":1779306309834,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":28,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,2,2]],"date-time":"2018-02-02T00:00:00Z","timestamp":1517529600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,2,2]]},"DOI":"10.1145\/3159652.3159687","type":"proceedings-article","created":{"date-parts":[[2018,2,6]],"date-time":"2018-02-06T18:12:23Z","timestamp":1517940743000},"page":"198-206","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":128,"title":["Offline A\/B Testing for Recommender Systems"],"prefix":"10.1145","author":[{"given":"Alexandre","family":"Gilotte","sequence":"first","affiliation":[{"name":"Criteo Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cl\u00e9ment","family":"Calauz\u00e8nes","sequence":"additional","affiliation":[{"name":"Criteo Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thomas","family":"Nedelec","sequence":"additional","affiliation":[{"name":"Criteo Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexandre","family":"Abraham","sequence":"additional","affiliation":[{"name":"Criteo Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Simon","family":"Doll\u00e9","sequence":"additional","affiliation":[{"name":"Criteo Research, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2018,2,2]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Ricardo Baeza-Yates Berthier Ribeiro-Neto etal 1999. Modern information retrieval.   Ricardo Baeza-Yates Berthier Ribeiro-Neto et al. 1999. Modern information retrieval."},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of Journal of Machine Learning Research(JMLR).","author":"Bottou L\u00e9on","year":"2013","unstructured":"L\u00e9on Bottou and Jonas Peters . 2013 . Counterfactual reasoning and learning systems: the example of computational advertising . Proceedings of Journal of Machine Learning Research(JMLR). L\u00e9on Bottou and Jonas Peters. 2013. Counterfactual reasoning and learning systems: the example of computational advertising. Proceedings of Journal of Machine Learning Research(JMLR)."},{"key":"e_1_3_2_1_3_1","volume-title":"Terry","author":"Bradley Ralph Allan","year":"1952","unstructured":"Ralph Allan Bradley and Milton E . Terry . 1952 . Rank Analysis of Incomplete Block Designs: The Method of Paired Comparisons. Biometrika( 1952). Ralph Allan Bradley and Milton E. Terry. 1952. Rank Analysis of Incomplete Block Designs: The Method of Paired Comparisons. Biometrika(1952)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/3104322.3104351"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 28th International Conference on Machine Learning(ICML).","author":"Dudik Miroslav","year":"2011","unstructured":"Miroslav Dudik , John Langford , and Lihong Li . 2011 . Doubly robust policy evaluation and learning . Proceedings of the 28th International Conference on Machine Learning(ICML). Miroslav Dudik, John Langford, and Lihong Li. 2011. Doubly robust policy evaluation and learning. Proceedings of the 28th International Conference on Machine Learning(ICML)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553423"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"JM Hammersley and DC Handscomb. 1964. Monte Carlo Methods. Chapter.  JM Hammersley and DC Handscomb. 1964. Monte Carlo Methods. Chapter.","DOI":"10.1007\/978-94-009-5819-7"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2827872"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/963770.963772"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1952.10483446"},{"key":"e_1_3_2_1_12_1","unstructured":"Business Insider. 2017. Morgan Stanley puts Amazon Prime subscribers at 65M. http:\/\/www.businessinsider.fr\/us\/morgan-stanley-puts-amazon-primesubscribers-at-65m-2017--2\/.(2017).  Business Insider. 2017. Morgan Stanley puts Amazon Prime subscribers at 65M. http:\/\/www.businessinsider.fr\/us\/morgan-stanley-puts-amazon-primesubscribers-at-65m-2017--2\/.(2017)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345545"},{"key":"e_1_3_2_1_14_1","unstructured":"Deba B Lahiri. 1951. A method of sample selection providing unbiased ratio estimates. Bulletin of the International Statistical Institute.  Deba B Lahiri. 1951. A method of sample selection providing unbiased ratio estimates. Bulletin of the International Statistical Institute."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935878"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5555\/3020488.3020521"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 22nd Annual Conference on Learning Theory(COLT).","author":"Maurer Andreas","year":"2009","unstructured":"Andreas Maurer and Massimiliano Pontil . 2009 . Empirical Bernstein bounds and sample variance penalization . Proceedings of the 22nd Annual Conference on Learning Theory(COLT). Andreas Maurer and Massimiliano Pontil. 2009. Empirical Bernstein bounds and sample variance penalization. Proceedings of the 22nd Annual Conference on Learning Theory(COLT)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02949779"},{"key":"e_1_3_2_1_19_1","unstructured":"Art Owen. 2010. Monte Carlo theory methods and examples.(2010). arXiv:arXiv:1012.5461v2  Art Owen. 2010. Monte Carlo theory methods and examples.(2010). arXiv:arXiv:1012.5461v2"},{"key":"e_1_3_2_1_20_1","article-title":"The Analysis of Permutations","author":"Plackett R. L.","year":"1975","unstructured":"R. L. Plackett . 1975 . The Analysis of Permutations . Journal of the Royal Statistical Society. Series C(Applied Statistics). R. L. Plackett. 1975. The Analysis of Permutations. Journal of the Royal Statistical Society. Series C(Applied Statistics).","journal-title":"Journal of the Royal Statistical Society. Series C(Applied Statistics)."},{"key":"e_1_3_2_1_21_1","article-title":"Weighted uniform sampling: a Monte Carlo technique for reducing variance","author":"Powell MJD","year":"1966","unstructured":"MJD Powell and J Swann . 1966 . Weighted uniform sampling: a Monte Carlo technique for reducing variance . Journal of Applied Mathematics. MJD Powell and J Swann. 1966. Weighted uniform sampling: a Monte Carlo technique for reducing variance. Journal of Applied Mathematics.","journal-title":"Journal of Applied Mathematics."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2365952.2365982"},{"key":"e_1_3_2_1_23_1","unstructured":"Amode Ranjan Sen. 1952. Present status of probability sampling and its use in estimation of farm characteristics. Econometrica(1952).  Amode Ranjan Sen. 1952. Present status of probability sampling and its use in estimation of farm characteristics. Econometrica(1952)."},{"key":"e_1_3_2_1_24_1","unstructured":"Adith Swaminathan and Thorsten Joachims. 2015. The Self-Normalized Estimator for Counterfactual Learning. Proceeding of Neural Information Processing Systems(NIPS).   Adith Swaminathan and Thorsten Joachims. 2015. The Self-Normalized Estimator for Counterfactual Learning. Proceeding of Neural Information Processing Systems(NIPS)."},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the International Conference on Machine Learning(ICML).","author":"Thomas Philip","year":"2016","unstructured":"Philip Thomas and Emma Brunskill . 2016 . Data-efficient off-policy policy evaluation for reinforcement learning . Proceedings of the International Conference on Machine Learning(ICML). Philip Thomas and Emma Brunskill. 2016. Data-efficient off-policy policy evaluation for reinforcement learning. Proceedings of the International Conference on Machine Learning(ICML)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Louis Leon Thurstone. 1927. A Law of Comparative Judgement. Psychological Review.  Louis Leon Thurstone. 1927. A Law of Comparative Judgement. Psychological Review.","DOI":"10.1037\/h0070288"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992696"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/0022-2496(77)90026-8"}],"event":{"name":"WSDM 2018: The Eleventh ACM International Conference on Web Search and Data Mining","location":"Marina Del Rey CA USA","acronym":"WSDM 2018","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3159652.3159687","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3159652.3159687","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T02:26:12Z","timestamp":1750213572000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3159652.3159687"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,2,2]]},"references-count":28,"alternative-id":["10.1145\/3159652.3159687","10.1145\/3159652"],"URL":"https:\/\/doi.org\/10.1145\/3159652.3159687","relation":{},"subject":[],"published":{"date-parts":[[2018,2,2]]},"assertion":[{"value":"2018-02-02","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}