{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:27:31Z","timestamp":1750220851758,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,1,20]],"date-time":"2020-01-20T00:00:00Z","timestamp":1579478400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,1,20]]},"DOI":"10.1145\/3336191.3371871","type":"proceedings-article","created":{"date-parts":[[2020,1,22]],"date-time":"2020-01-22T19:08:16Z","timestamp":1579720096000},"page":"877-880","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments"],"prefix":"10.1145","author":[{"given":"Somit","family":"Gupta","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaolin","family":"Shi","sequence":"additional","affiliation":[{"name":"Snap Inc., Santa Monica, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pavel","family":"Dmitriev","sequence":"additional","affiliation":[{"name":"Outreach, Seattle, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xin","family":"Fu","sequence":"additional","affiliation":[{"name":"Facebook, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Avijit","family":"Mukherjee","sequence":"additional","affiliation":[{"name":"Facebook, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,1,22]]},"reference":[{"volume-title":"Accelerating software innovation: Big data conference & machine learning training | Strata Data","year":"2018","key":"e_1_3_2_1_1_1","unstructured":"A\/B testing at scale : Accelerating software innovation: Big data conference & machine learning training | Strata Data : 2018 . https:\/\/conferences.oreilly.com\/strata\/strata-ca-2018\/public\/schedule\/detail\/63322. Accessed : 2019-02--18. A\/B testing at scale: Accelerating software innovation: Big data conference & machine learning training | Strata Data: 2018. https:\/\/conferences.oreilly.com\/strata\/strata-ca-2018\/public\/schedule\/detail\/63322. Accessed: 2019-02--18."},{"key":"e_1_3_2_1_2_1","unstructured":"Advanced Topics in Online Experiments -- ExP Platform: https:\/\/exp-platform.com\/advanced-topics-in-online-experiments\/. Accessed: 2019-09-09.  Advanced Topics in Online Experiments -- ExP Platform: https:\/\/exp-platform.com\/advanced-topics-in-online-experiments\/. Accessed: 2019-09-09."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3082060"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16","author":"Deng A.","year":"2016","unstructured":"Deng , A. and Shi , X . 2016. Data-Driven Metric Development for Online Controlled Experiments . Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16 ( New York, New York, USA , 2016 ), 77--86. Deng, A. and Shi, X. 2016. Data-Driven Metric Development for Online Controlled Experiments. Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16 (New York, New York, USA, 2016), 77--86."},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16","author":"Deng A.","year":"2016","unstructured":"Deng , A. and Shi , X . 2016. Data-Driven Metric Development for Online Controlled Experiments . Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16 ( 2016 ), 77--86. Deng, A. and Shi, X. 2016. Data-Driven Metric Development for Online Controlled Experiments. Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16 (2016), 77--86."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098024"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840744"},{"key":"e_1_3_2_1_9_1","volume-title":"Measuring Metrics. Proceedings of the 25th ACM International on Conference on Information and Knowledge Management - CIKM '16","author":"Dmitriev P.","year":"2016","unstructured":"Dmitriev , P. and Wu , X . 2016 . Measuring Metrics. Proceedings of the 25th ACM International on Conference on Information and Knowledge Management - CIKM '16 ( 2016 ), 429--437. Dmitriev, P. and Wu, X. 2016. Measuring Metrics. Proceedings of the 25th ACM International on Conference on Information and Knowledge Management - CIKM '16 (2016), 429--437."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining - KDD '19","author":"Fabijan A.","year":"2019","unstructured":"Fabijan , A. et al. 2019. Diagnosing Sample Ratio Mismatch in Online Controlled Experiments . Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining - KDD '19 ( New York, New York, USA , 2019 ), 2156--2164. Fabijan, A. et al. 2019. Diagnosing Sample Ratio Mismatch in Online Controlled Experiments. Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining - KDD '19 (New York, New York, USA, 2019), 2156--2164."},{"key":"e_1_3_2_1_11_1","volume-title":"et al","author":"Fabijan A.","year":"2019","unstructured":"Fabijan , A. et al . 2019 . Three Key Checklists and Remedies for Trustworthy Analysis of Online Controlled Experiments at Scale . to appear in the proceedings of 2019 IEEE\/ACM 39th International Conference on Software Engineering (ICSE) Software Engineering in Practice (SEIP) (Montreal, Canada, 2019). Fabijan, A. et al. 2019. Three Key Checklists and Remedies for Trustworthy Analysis of Online Controlled Experiments at Scale. to appear in the proceedings of 2019 IEEE\/ACM 39th International Conference on Software Engineering (ICSE) Software Engineering in Practice (SEIP) (Montreal, Canada, 2019)."},{"key":"e_1_3_2_1_12_1","volume-title":"Turin","author":"Gupchup J.","year":"2018","unstructured":"Gupchup , J. et al. 2018. Trustworthy Experimentation Under Telemetry Loss. to appear in: Proceedings of the 27th ACM International on Conference on Information and Knowledge Management - CIKM '18 (Lingotto , Turin , 2018 ). Gupchup, J. et al. 2018. Trustworthy Experimentation Under Telemetry Loss. to appear in: Proceedings of the 27th ACM International on Conference on Information and Knowledge Management - CIKM '18 (Lingotto, Turin, 2018)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3320093"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331651.3331655"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 22nd ACM international conference on Conference on information & knowledge management - CIKM '13","author":"Hassan A.","year":"2013","unstructured":"Hassan , A. et al. 2013. Beyond clicks . Proceedings of the 22nd ACM international conference on Conference on information & knowledge management - CIKM '13 ( New York, New York, USA , 2013 ), 2019--2028. Hassan, A. et al. 2013. Beyond clicks. Proceedings of the 22nd ACM international conference on Conference on information & knowledge management - CIKM '13 (New York, New York, USA, 2013), 2019--2028."},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD '14","author":"Kohavi R.","year":"2014","unstructured":"Kohavi , R. et al. 2014. Seven rules of thumb for web site experimenters . Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD '14 ( New York, USA , 2014 ), 1857--1866. Kohavi, R. et al. 2014. Seven rules of thumb for web site experimenters. Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining - KDD '14 (New York, USA, 2014), 1857--1866."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 2017 ACM on Conference on Information and Knowledge Management - CIKM '17","author":"Machmouchi W.","year":"2017","unstructured":"Machmouchi , W. et al. 2017. Beyond Success Rate . Proceedings of the 2017 ACM on Conference on Information and Knowledge Management - CIKM '17 ( New York, New York, USA , 2017 ), 757--765. Machmouchi, W. et al. 2017. Beyond Success Rate. Proceedings of the 2017 ACM on Conference on Information and Knowledge Management - CIKM '17 (New York, New York, USA, 2017), 757--765."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926731"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939688"},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems.","author":"Rodden K.","year":"2010","unstructured":"Rodden , K. et al. 2010. Measuring the User Experience on a Large Scale?: User-Centered Metrics for Web Applications . Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. ( 2010 ), 2395--2398. DOI:https:\/\/doi.org\/10.1145\/1753326.1753687. 10.1145\/1753326.1753687 Rodden, K. et al. 2010. Measuring the User Experience on a Large Scale?: User-Centered Metrics for Web Applications. Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. (2010), 2395--2398. DOI:https:\/\/doi.org\/10.1145\/1753326.1753687."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 28th international conference on Human factors in computing systems - CHI '10","author":"Rodden K.","year":"2010","unstructured":"Rodden , K. et al. 2010. Measuring the user experience on a large scale . Proceedings of the 28th international conference on Human factors in computing systems - CHI '10 ( New York, New York, USA , 2010 ), 2395. Rodden, K. et al. 2010. Measuring the user experience on a large scale. Proceedings of the 28th international conference on Human factors in computing systems - CHI '10 (New York, New York, USA, 2010), 2395."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3332297"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16.","author":"Xie H.","year":"2016","unstructured":"Xie , H. and Aurisset , J . 2016. Improving the Sensitivity of Online Controlled Experiments . Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16. ( 2016 ), 645--654. DOI:https:\/\/doi.org\/10.1145\/2939672.2939733. 10.1145\/2939672.2939733 Xie, H. and Aurisset, J. 2016. Improving the Sensitivity of Online Controlled Experiments. Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining - KDD '16. (2016), 645--654. DOI:https:\/\/doi.org\/10.1145\/2939672.2939733."},{"key":"e_1_3_2_1_25_1","volume-title":"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct.","author":"Zhao Z.","year":"2016","unstructured":"Zhao , Z. et al. 2016 . Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct. 2016 ), 498--507. Zhao, Z. et al. 2016. Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct. 2016), 498--507."}],"event":{"name":"WSDM '20: The Thirteenth ACM International Conference on Web Search and Data Mining","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Houston TX USA","acronym":"WSDM '20"},"container-title":["Proceedings of the 13th International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3336191.3371871","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3336191.3371871","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:23:15Z","timestamp":1750202595000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3336191.3371871"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,1,20]]},"references-count":25,"alternative-id":["10.1145\/3336191.3371871","10.1145\/3336191"],"URL":"https:\/\/doi.org\/10.1145\/3336191.3371871","relation":{},"subject":[],"published":{"date-parts":[[2020,1,20]]},"assertion":[{"value":"2020-01-22","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}