{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:10:11Z","timestamp":1750201811310,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,4,20]],"date-time":"2020-04-20T00:00:00Z","timestamp":1587340800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,4,20]]},"DOI":"10.1145\/3366424.3383117","type":"proceedings-article","created":{"date-parts":[[2020,5,4]],"date-time":"2020-05-04T08:10:56Z","timestamp":1588579856000},"page":"317-319","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments"],"prefix":"10.1145","author":[{"given":"Somit","family":"Gupta","sequence":"first","affiliation":[{"name":"Microsoft Redmond, WA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaolin","family":"Shi","sequence":"additional","affiliation":[{"name":"Snap Inc. Santa Monica, CA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pavel","family":"Dmitriev","sequence":"additional","affiliation":[{"name":"Outreach Seattle, WA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xin","family":"Fu","sequence":"additional","affiliation":[{"name":"Facebook Menlo Park, CA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,4,20]]},"reference":[{"volume-title":"Accelerating software innovation: Big data conference & machine learning training | Strata Data","year":"2018","key":"e_1_3_2_1_1_1"},{"key":"e_1_3_2_1_2_1","unstructured":"Advanced Topics in Online Experiments \u2013 ExP Platform: https:\/\/exp-platform.com\/advanced-topics-in-online-experiments\/. Accessed: 2019-09-09.  Advanced Topics in Online Experiments \u2013 ExP Platform: https:\/\/exp-platform.com\/advanced-topics-in-online-experiments\/. Accessed: 2019-09-09."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3082060"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"},{"volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD \u201916","year":"2016","author":"Deng A.","key":"e_1_3_2_1_5_1"},{"volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD \u201916","year":"2016","author":"Deng A.","key":"e_1_3_2_1_6_1"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098024"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840744"},{"volume-title":"Measuring Metrics. Proceedings of the 25th ACM International on Conference on Information and Knowledge Management - CIKM \u201916","year":"2016","author":"Dmitriev P.","key":"e_1_3_2_1_9_1"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330722"},{"volume-title":"Canada","year":"2019","author":"Fabijan A.","key":"e_1_3_2_1_11_1"},{"volume-title":"Turin","year":"2018","author":"Gupchup J.","key":"e_1_3_2_1_12_1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3320093"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331651.3331655"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505682"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3132850"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926731"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939688"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753687"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753687"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3332297"},{"volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD \u201916","year":"2016","author":"Xie H.","key":"e_1_3_2_1_24_1"},{"volume-title":"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct.","year":"2016","author":"Zhao Z.","key":"e_1_3_2_1_25_1"}],"event":{"name":"WWW '20: The Web Conference 2020","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Taipei Taiwan","acronym":"WWW '20"},"container-title":["Companion Proceedings of the Web Conference 2020"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3366424.3383117","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3366424.3383117","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:33:06Z","timestamp":1750199586000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3366424.3383117"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,4,20]]},"references-count":25,"alternative-id":["10.1145\/3366424.3383117","10.1145\/3366424"],"URL":"https:\/\/doi.org\/10.1145\/3366424.3383117","relation":{},"subject":[],"published":{"date-parts":[[2020,4,20]]},"assertion":[{"value":"2020-04-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}