{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:30:07Z","timestamp":1750221007077,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,25]],"date-time":"2019-07-25T00:00:00Z","timestamp":1564012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,25]]},"DOI":"10.1145\/3292500.3332297","type":"proceedings-article","created":{"date-parts":[[2019,7,26]],"date-time":"2019-07-26T13:17:26Z","timestamp":1564147046000},"page":"3189-3190","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Challenges, Best Practices and Pitfalls in Evaluating Results of Online Controlled Experiments"],"prefix":"10.1145","author":[{"given":"Xiaolin","family":"Shi","sequence":"first","affiliation":[{"name":"Snap Inc., Santa Monica, CA, USA"}]},{"given":"Pavel","family":"Dmitriev","sequence":"additional","affiliation":[{"name":"Outreach, Seattle, WA, USA"}]},{"given":"Somit","family":"Gupta","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Xin","family":"Fu","sequence":"additional","affiliation":[{"name":"Facebook, Menlo Park, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2019,7,25]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"A\/B Testing at Scale Tutorial Strata 2018: https:\/\/exp-platform.com\/2018StrataABtutorial\/. Accessed: 2019-02-05.  A\/B Testing at Scale Tutorial Strata 2018: https:\/\/exp-platform.com\/2018StrataABtutorial\/. Accessed: 2019-02-05."},{"key":"e_1_3_2_2_2_1","unstructured":"AdvancedTopic_SpeedMatters.docx - Microsoft Word Online: https:\/\/onedrive.live.com\/view.aspx?resid=8612090E610871E4!286184&ithint=file%2Cdocx&app=Word&authkey=!AGm2aNDnCqKOsYc. Accessed: 2019-02--20.  AdvancedTopic_SpeedMatters.docx - Microsoft Word Online: https:\/\/onedrive.live.com\/view.aspx?resid=8612090E610871E4!286184&ithint=file%2Cdocx&app=Word&authkey=!AGm2aNDnCqKOsYc. Accessed: 2019-02--20."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330722"},{"volume-title":"Online Evaluation for Effective Web Service Development. WWW 2018 (Sep.","year":"2018","author":"Budylin R.","key":"e_1_3_2_2_4_1"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3041021.3054199"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3082060"},{"volume-title":"et al","year":"2016","author":"Deng A.","key":"e_1_3_2_2_8_1"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2742563"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098024"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840744"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983356"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2018.00021"},{"volume-title":"et al","year":"2018","author":"Fabijan A.","key":"e_1_3_2_2_17_1"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00009"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2789994"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271747"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3320093"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSA.2018.00009"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331651.3331655"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505682"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788583"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685319"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018708"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2785464"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926731"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753687"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488494"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3219860"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788602"},{"volume-title":"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct.","year":"2016","author":"Zhao Z.","key":"e_1_3_2_2_35_1"}],"event":{"name":"KDD '19: The 25th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"],"location":"Anchorage AK USA","acronym":"KDD '19"},"container-title":["Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3292500.3332297","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3292500.3332297","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T00:25:56Z","timestamp":1750206356000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3292500.3332297"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,7,25]]},"references-count":35,"alternative-id":["10.1145\/3292500.3332297","10.1145\/3292500"],"URL":"https:\/\/doi.org\/10.1145\/3292500.3332297","relation":{},"subject":[],"published":{"date-parts":[[2019,7,25]]},"assertion":[{"value":"2019-07-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}