{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,10]],"date-time":"2026-03-10T12:16:04Z","timestamp":1773144964149,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,2,2]],"date-time":"2017-02-02T00:00:00Z","timestamp":1485993600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,2,2]]},"DOI":"10.1145\/3018661.3018677","type":"proceedings-article","created":{"date-parts":[[2017,2,2]],"date-time":"2017-02-02T20:09:48Z","timestamp":1486066188000},"page":"641-649","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":25,"title":["Trustworthy Analysis of Online A\/B Tests"],"prefix":"10.1145","author":[{"given":"Alex","family":"Deng","sequence":"first","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}]},{"given":"Jiannan","family":"Lu","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}]},{"given":"Jonthan","family":"Litz","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}]}],"member":"320","published-online":{"date-parts":[[2017,2,2]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Athey S. and Imbens G. [2016] 'The econometrics of randomized experiments' arXiv:1607.00698 .  Athey S. and Imbens G. [2016] 'The econometrics of randomized experiments' arXiv:1607.00698 .","DOI":"10.1016\/bs.hefe.2016.10.003"},{"key":"e_1_3_2_1_2_1","first-page":"1303","volume-title":"19th ACM SIGKDD Int. Conf. Knowl. Discov. data Min.', ACM","author":"Bakshy E."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567967"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Barber D. [2012] Bayesian reasoning and machine learning Cambridge University Press.  Barber D. [2012] Bayesian reasoning and machine learning Cambridge University Press.","DOI":"10.1017\/CBO9780511804779"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.2015.1089790"},{"key":"e_1_3_2_1_6_1","unstructured":"Cochran W. G. [1977] Sampling Techniques Third Edition New York: W.W. Norton.  Cochran W. G. [1977] Sampling Techniques Third Edition New York: W.W. Norton."},{"key":"e_1_3_2_1_7_1","unstructured":"DasGupta A. [2008] Asymptotic Theory of Statistics and Probability Springer.  DasGupta A. [2008] Asymptotic Theory of Statistics and Probability Springer."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_1_10_1","volume-title":"JSM Proc. .","author":"Deng S."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Efron B. and Tibshirani R. J. [1994] An introduction to the bootstrap CRC press.  Efron B. and Tibshirani R. J. [1994] An introduction to the bootstrap CRC press.","DOI":"10.1201\/9780429246593"},{"key":"e_1_3_2_1_12_1","unstructured":"Fisher R. A. [1925] Statistical Methods for Research Workers First Edition Edinburgh: Oliver and Boyd. [13] Fisher R. A. [1935] The Design of Experiments First Edition Edinburgh: Oliver and Boyd.  Fisher R. A. [1925] Statistical Methods for Research Workers First Edition Edinburgh: Oliver and Boyd. [13] Fisher R. A. [1935] The Design of Experiments First Edition Edinburgh: Oliver and Boyd."},{"issue":"4","key":"e_1_3_2_1_13_1","first-page":"13","article-title":"], 'The netflix recommender system: Algorithms, business value, and innovation","volume":"6","author":"Gomez-Uribe C. A.","year":"2016","journal-title":"ACM Transactions on Management Information Systems (TMIS)"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.2015.1089789"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139025751"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Kohavi R. and Longbotham R. [2015] 'Online controlled experiments and A\/B tests' Encyclopedia of Meaning Learning and Data Mining .  Kohavi R. and Longbotham R. [2015] 'Online controlled experiments and A\/B tests' Encyclopedia of Meaning Learning and Data Mining .","DOI":"10.1007\/978-1-4899-7502-7_891-1"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2010.264"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1214\/12-AOAS583"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.spl.2016.06.007"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1214\/12-AOS1008"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2015.1079528"},{"key":"e_1_3_2_1_24_1","unstructured":"Murphy K. P. [2012] Machine learning: a probabilistic perspective MIT press.  Murphy K. P. [2012] Machine learning: a probabilistic perspective MIT press."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1214\/ss\/1177012031"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1214\/07-AOAS122"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0037350"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.2307\/2287653"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1214\/08-AOAS187"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177730256"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2815400.2815401"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Wasserman L. [2003] All of Statistics: A Concise Course in Statistical Inference Springer.  Wasserman L. [2003] All of Statistics: A Concise Course in Statistical Inference Springer.","DOI":"10.1007\/978-0-387-21736-9"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788602"}],"event":{"name":"WSDM 2017: Tenth ACM International Conference on Web Search and Data Mining","location":"Cambridge United Kingdom","acronym":"WSDM 2017","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the Tenth ACM International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3018661.3018677","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3018661.3018677","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:23:59Z","timestamp":1750220639000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3018661.3018677"}},"subtitle":["Pitfalls, challenges and solutions"],"short-title":[],"issued":{"date-parts":[[2017,2,2]]},"references-count":34,"alternative-id":["10.1145\/3018661.3018677","10.1145\/3018661"],"URL":"https:\/\/doi.org\/10.1145\/3018661.3018677","relation":{},"subject":[],"published":{"date-parts":[[2017,2,2]]},"assertion":[{"value":"2017-02-02","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}