{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,30]],"date-time":"2026-07-30T02:10:10Z","timestamp":1785377410892,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,25]],"date-time":"2019-07-25T00:00:00Z","timestamp":1564012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,25]]},"DOI":"10.1145\/3292500.3330722","type":"proceedings-article","created":{"date-parts":[[2019,7,26]],"date-time":"2019-07-26T13:17:26Z","timestamp":1564147046000},"page":"2156-2164","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":24,"title":["Diagnosing Sample Ratio Mismatch in Online Controlled Experiments"],"prefix":"10.1145","author":[{"given":"Aleksander","family":"Fabijan","sequence":"first","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jayant","family":"Gupchup","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Somit","family":"Gupta","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeff","family":"Omhover","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wen","family":"Qin","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lukas","family":"Vermeer","sequence":"additional","affiliation":[{"name":"Booking.com, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pavel","family":"Dmitriev","sequence":"additional","affiliation":[{"name":"Outreach.io, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2019,7,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2018.00062"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567967"},{"key":"e_1_3_2_1_3_1","volume-title":"System: A Case Study of Logging and Telemetry at Microsoft.","author":"Barik T.","year":"2016"},{"key":"e_1_3_2_1_4_1","unstructured":"Chen N. Liu M. and Xu Y. 2018. Automatic Detection and Diagnosis of Biased Online Experiments. arXiv preprint arXiv:1808.00114. (2018).  Chen N. Liu M. and Xu Y. 2018. Automatic Detection and Diagnosis of Biased Online Experiments. arXiv preprint arXiv:1808.00114. (2018)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018661.3018677"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Devore J.L. and Berk K.N. 2012. Modern Mathematical Statistics with Applications.  Devore J.L. and Berk K.N. 2012. Modern Mathematical Statistics with Applications.","DOI":"10.1007\/978-1-4614-0391-3"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840744"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098024"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2018.00020"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1002\/smr.2113"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2018.00021"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/SEAA.2017.47"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271747"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSA.2018.00009"},{"key":"e_1_3_2_1_15_1","unstructured":"Kaufman R.L. Pitchforth J. and Vermeer L. 2017. Democratizing online controlled experiments at Booking. com. arXiv preprint arXiv:1710.08217. (2017) 1--7.  Kaufman R.L. Pitchforth J. and Vermeer L. 2017. Democratizing online controlled experiments at Booking. com. arXiv preprint arXiv:1710.08217. (2017) 1--7."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2017.19"},{"key":"e_1_3_2_1_17_1","unstructured":"Kluck T. and Vermeer L. 2017. Leaky Abstraction In Online Experimentation Platforms: A Conceptual Framework To Categorize Common Challenges. (Oct. 2017).  Kluck T. and Vermeer L. 2017. Leaky Abstraction In Online Experimentation Platforms: A Conceptual Framework To Categorize Common Challenges. (Oct. 2017)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Kohavi R. and Longbotham R. 2017. Online Controlled Experiments and A\/B Testing. Encyclopedia of Machine Learning and Data Mining. C. Sammut and G.I. Webb eds. Springer US. 922--929.  Kohavi R. and Longbotham R. 2017. Online Controlled Experiments and A\/B Testing. Encyclopedia of Machine Learning and Data Mining. C. Sammut and G.I. Webb eds. Springer US. 922--929.","DOI":"10.1007\/978-1-4899-7687-1_891"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_22_1","unstructured":"Kohavi R. and Thomke S. 2017. The Surprising Power of Online Experiments. Harvard Business Review.  Kohavi R. and Thomke S. 2017. The Surprising Power of Online Experiments. Harvard Business Review."},{"key":"e_1_3_2_1_23_1","unstructured":"Leaky Abstractions: 2018. https:\/\/booking.ai\/leaky-abstractions-in-online-experimentation-platforms-ae4cf05013f9.  Leaky Abstractions: 2018. https:\/\/booking.ai\/leaky-abstractions-in-online-experimentation-platforms-ae4cf05013f9."},{"key":"e_1_3_2_1_24_1","volume-title":"Lecture Notes in Business Information Processing (Cham","author":"Lindgren E.","year":"2015"},{"key":"e_1_3_2_1_25_1","unstructured":"List of browsers that support 128-bit and 256-bit encryption:.  List of browsers that support 128-bit and 256-bit encryption:."},{"key":"e_1_3_2_1_26_1","unstructured":"Mayring P. 2002. Qualitative content analysis - research instrument or mode of interpretation. The Role of the Researcher in Qualitative Psychology. 139--148.  Mayring P. 2002. Qualitative content analysis - research instrument or mode of interpretation. The Role of the Researcher in Qualitative Psychology. 139--148."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.2307\/2315408"},{"key":"e_1_3_2_1_28_1","unstructured":"Microsoft Experimentation Platform: http:\/\/www.exp-platform.com.  Microsoft Experimentation Platform: http:\/\/www.exp-platform.com."},{"key":"e_1_3_2_1_29_1","unstructured":"MIT Code: 2016. http:\/\/bit.ly\/Code2016Kohavi.  MIT Code: 2016. http:\/\/bit.ly\/Code2016Kohavi."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1080\/14786440009463897"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"e_1_3_2_1_32_1","unstructured":"SRM Interview Guide: 2019. https:\/\/www.dropbox.com\/s\/h0291u1fcqg0eze\/SRM Interview Guide.pdf?dl=0.  SRM Interview Guide: 2019. https:\/\/www.dropbox.com\/s\/h0291u1fcqg0eze\/SRM Interview Guide.pdf?dl=0."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788602"},{"key":"e_1_3_2_1_35_1","volume-title":"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation. 2016 IEEE International Conference on Data Science and Advanced Analytics (DSAA) (Oct.","author":"Zhao Z.","year":"2016"}],"event":{"name":"KDD '19: The 25th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Anchorage AK USA","acronym":"KDD '19","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3292500.3330722","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3292500.3330722","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T00:57:50Z","timestamp":1750208270000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3292500.3330722"}},"subtitle":["A Taxonomy and Rules of Thumb for Practitioners"],"short-title":[],"issued":{"date-parts":[[2019,7,25]]},"references-count":35,"alternative-id":["10.1145\/3292500.3330722","10.1145\/3292500"],"URL":"https:\/\/doi.org\/10.1145\/3292500.3330722","relation":{},"subject":[],"published":{"date-parts":[[2019,7,25]]},"assertion":[{"value":"2019-07-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}