{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T02:41:05Z","timestamp":1783737665563,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,13]],"date-time":"2017-08-13T00:00:00Z","timestamp":1502582400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,13]]},"DOI":"10.1145\/3097983.3098024","type":"proceedings-article","created":{"date-parts":[[2017,8,4]],"date-time":"2017-08-04T18:35:54Z","timestamp":1501871754000},"page":"1427-1436","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":83,"title":["A Dirty Dozen"],"prefix":"10.1145","author":[{"given":"Pavel","family":"Dmitriev","sequence":"first","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Somit","family":"Gupta","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dong Woo","family":"Kim","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Garnet","family":"Vaz","sequence":"additional","affiliation":[{"name":"Microsoft Corporation, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2017,8,13]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926731"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983356"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1053\/j.seminhematol.2008.04.003"},{"key":"e_1_3_2_2_5_1","volume-title":"Online Controlled Experiments and A\/B Tests.,\" in Encyclopedia of Machine Learning and Data Mining","author":"Kohavi R.","year":"2017","unstructured":"R. Kohavi and R. Longbotham , \" Online Controlled Experiments and A\/B Tests.,\" in Encyclopedia of Machine Learning and Data Mining , 2017 . R. Kohavi and R. Longbotham, \"Online Controlled Experiments and A\/B Tests.,\" in Encyclopedia of Machine Learning and Data Mining, 2017."},{"key":"e_1_3_2_2_6_1","unstructured":"\"Microsoft Experimentation Platform \" [Online]. Available: http:\/\/www.exp-platform.com.  \"Microsoft Experimentation Platform \" [Online]. Available: http:\/\/www.exp-platform.com."},{"key":"e_1_3_2_2_7_1","volume-title":"Online Controlled Experiments at Large Scale,\" in KDD","author":"Kohavi R.","year":"2013","unstructured":"R. Kohavi , A. Deng , B. Frasca , T. Walker , Y. Xu and N. Pohlmann , \" Online Controlled Experiments at Large Scale,\" in KDD , 2013 . R. Kohavi, A. Deng, B. Frasca, T. Walker, Y. Xu and N. Pohlmann, \"Online Controlled Experiments at Large Scale,\" in KDD, 2013."},{"key":"e_1_3_2_2_8_1","volume-title":"Seven Rules of Thumb for Web Site Experimenters,\" in KDD","author":"Kohavi R.","year":"2014","unstructured":"R. Kohavi , A. Deng , R. Longbotham and Y. Xu , \" Seven Rules of Thumb for Web Site Experimenters,\" in KDD , 2014 . R. Kohavi, A. Deng, R. Longbotham and Y. Xu, \"Seven Rules of Thumb for Web Site Experimenters,\" in KDD, 2014."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840744"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788583"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.2307\/2390989"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1108\/01443579410072391"},{"key":"e_1_3_2_2_15_1","article-title":"The Balanced Scorecard - Measures that Drive Performance","author":"Kaplan R. S.","year":"1992","unstructured":"R. S. Kaplan and D. Norton , \" The Balanced Scorecard - Measures that Drive Performance ,\" Harvard Business Review , 1992 . R. S. Kaplan and D. Norton, \"The Balanced Scorecard - Measures that Drive Performance,\" Harvard Business Review, 1992.","journal-title":"Harvard Business Review"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0263-2373(98)00029-2"},{"key":"e_1_3_2_2_17_1","volume-title":"Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA","author":"Deng A.","year":"2016","unstructured":"A. Deng , J. Lu and S. Chen , \" Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA , 2016 . A. Deng, J. Lu and S. Chen, \"Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA, 2016."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2742563"},{"key":"e_1_3_2_2_19_1","volume-title":"Always valid inference: Bringing sequential analysis to A\/B testing.,\" In submission. Preprint available at arxiv.org\/pdf\/1512.04922","author":"Johari R.","year":"2015","unstructured":"R. Johari , L. Pekelis and D. J. Walsh , \" Always valid inference: Bringing sequential analysis to A\/B testing.,\" In submission. Preprint available at arxiv.org\/pdf\/1512.04922 , 2015 . R. Johari, L. Pekelis and D. J. Walsh, \"Always valid inference: Bringing sequential analysis to A\/B testing.,\" In submission. Preprint available at arxiv.org\/pdf\/1512.04922, 2015."},{"key":"e_1_3_2_2_20_1","unstructured":"R. Kohavi \"Lessons from running thousands of A\/B tests \" 2014. [Online]. Available: http:\/\/bit.ly\/expLesssonsCode.  R. Kohavi \"Lessons from running thousands of A\/B tests \" 2014. [Online]. Available: http:\/\/bit.ly\/expLesssonsCode."},{"key":"e_1_3_2_2_21_1","volume-title":"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation,\" in DSAA","author":"Zhao Z.","year":"2016","unstructured":"Z. Zhao , M. Chen , D. Matheson and M. Stone , \" Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation,\" in DSAA , 2016 . Z. Zhao, M. Chen, D. Matheson and M. Stone, \"Online Experimentation Diagnosis and Troubleshooting Beyond AA Validation,\" in DSAA, 2016."},{"key":"e_1_3_2_2_22_1","volume-title":"Trustworthy online controlled experiments: Five puzzling outcomes explained,\" in KDD","author":"Kohavi R.","year":"2012","unstructured":"R. Kohavi , A. Deng , B. Frasca , R. Longbotham , T. Walker and Y. Xu , \" Trustworthy online controlled experiments: Five puzzling outcomes explained,\" in KDD , 2012 . R. Kohavi, A. Deng, B. Frasca, R. Longbotham, T. Walker and Y. Xu, \"Trustworthy online controlled experiments: Five puzzling outcomes explained,\" in KDD, 2012."},{"key":"e_1_3_2_2_23_1","unstructured":"Wikipedia [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Click-through_rate.  Wikipedia [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Click-through_rate."},{"key":"e_1_3_2_2_24_1","volume-title":"Definition,\" AdWords Google","year":"2016","unstructured":"\"Clickthrough rate (CTR) : Definition,\" AdWords Google , 2016 . [Online]. Available: https:\/\/support.google.com\/adwords\/answer\/2615875. \"Clickthrough rate (CTR): Definition,\" AdWords Google, 2016. [Online]. Available: https:\/\/support.google.com\/adwords\/answer\/2615875."},{"key":"e_1_3_2_2_25_1","volume-title":"Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss","author":"Kohavi R.","year":"2010","unstructured":"R. Kohavi , Messner, S. Eliot , J. L. Ferres , R. Henne , V. Kannappan and J. Wang , \" Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss ,\" October 2010 . [Online]. Available: http:\/\/bit.ly\/expTrackingClicks. R. Kohavi, Messner, S. Eliot, J. L. Ferres, R. Henne, V. Kannappan and J. Wang, \"Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss,\" October 2010. [Online]. Available: http:\/\/bit.ly\/expTrackingClicks."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1097\/EDE.0b013e31818131e7"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1257\/jep.2.1.191"},{"issue":"5","key":"e_1_3_2_2_28_1","first-page":"157","article-title":"On the criterion that a given system of deviations from the probable in the case of a correlated system of variables is such that it can be reasonably supposed to have arisen from random sampling","volume":"50","author":"Peasron K.","year":"1900","unstructured":"K. Peasron , \" On the criterion that a given system of deviations from the probable in the case of a correlated system of variables is such that it can be reasonably supposed to have arisen from random sampling ,\" Philosophical Magazine , vol. 50 , no. 5 , p. 157 -- 175 , 1900 . K. Peasron, \"On the criterion that a given system of deviations from the probable in the case of a correlated system of variables is such that it can be reasonably supposed to have arisen from random sampling,\" Philosophical Magazine, vol. 50, no. 5, p. 157--175, 1900.","journal-title":"Philosophical Magazine"},{"key":"e_1_3_2_2_29_1","volume-title":"Statistical Methods for Research Workers","author":"Fischer R.","year":"1925","unstructured":"R. Fischer , Statistical Methods for Research Workers , Edinburgh : Oliver & Boyd , 1925 . R. Fischer, Statistical Methods for Research Workers, Edinburgh: Oliver & Boyd, 1925."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.2016.1154108"},{"key":"e_1_3_2_2_31_1","unstructured":"\"Fisher's Method \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Fisher%27s_method .  \"Fisher's Method \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Fisher%27s_method ."},{"key":"e_1_3_2_2_32_1","volume-title":"Lessons from Running A\/B\/n Tests for 12 years","author":"Kohavi R.","year":"2015","unstructured":"R. Kohavi , \"Online Controlled Experiments : Lessons from Running A\/B\/n Tests for 12 years ,\" 2015 . [Online]. Available: http:\/\/bit.ly\/KDD2015Kohavi. R. Kohavi, \"Online Controlled Experiments: Lessons from Running A\/B\/n Tests for 12 years,\" 2015. [Online]. Available: http:\/\/bit.ly\/KDD2015Kohavi."},{"key":"e_1_3_2_2_33_1","volume-title":"Always valid inference: Bringing sequential analysis to A\/B testing","author":"Johari R.","year":"2015","unstructured":"R. Johari , P. Leo and J. W. David , \" Always valid inference: Bringing sequential analysis to A\/B testing ,\" 2015 . [Online]. Available: https:\/\/arxiv.org\/abs\/1512.04922. R. Johari, P. Leo and J. W. David, \"Always valid inference: Bringing sequential analysis to A\/B testing,\" 2015. [Online]. Available: https:\/\/arxiv.org\/abs\/1512.04922."},{"key":"e_1_3_2_2_34_1","volume-title":"Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA","author":"Deng A.","year":"2016","unstructured":"A. Deng , J. Lu and S. Chen , \" Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA , 2016 . A. Deng, J. Lu and S. Chen, \"Continuous monitoring of A\/B tests without pain: Optional stopping in Bayesian testing,\" in DSAA, 2016."},{"key":"e_1_3_2_2_35_1","unstructured":"A. Deng P. Zhang S. Chen D. Kim and J. Lu \"Concise Summarization of Heterogeneous Treatment Effect Using Total Variation Regularized Regression \" in In submission.  A. Deng P. Zhang S. Chen D. Kim and J. Lu \"Concise Summarization of Heterogeneous Treatment Effect Using Total Variation Regularized Regression \" in In submission."},{"key":"e_1_3_2_2_36_1","unstructured":"\"Simpson's paradox \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Simpson%27s_paradox.  \"Simpson's paradox \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Simpson%27s_paradox."},{"key":"e_1_3_2_2_37_1","unstructured":"\"Multiple Comparisons problem \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Multiple_comparisons_problem.  \"Multiple Comparisons problem \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Multiple_comparisons_problem."},{"key":"e_1_3_2_2_38_1","unstructured":"\"Bonferroni correction \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Bonferroni_correction].  \"Bonferroni correction \" [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/Bonferroni_correction]."},{"key":"e_1_3_2_2_39_1","unstructured":"\"Mobile Patterns \" [Online]. Available: https:\/\/mobilepatterns.wikispaces.com\/Coach+Marks.  \"Mobile Patterns \" [Online]. Available: https:\/\/mobilepatterns.wikispaces.com\/Coach+Marks."}],"event":{"name":"KDD '17: The 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","location":"Halifax NS Canada","acronym":"KDD '17","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3097983.3098024","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3097983.3098024","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:03:43Z","timestamp":1750273423000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3097983.3098024"}},"subtitle":["Twelve Common Metric Interpretation Pitfalls in Online Controlled Experiments"],"short-title":[],"issued":{"date-parts":[[2017,8,13]]},"references-count":39,"alternative-id":["10.1145\/3097983.3098024","10.1145\/3097983"],"URL":"https:\/\/doi.org\/10.1145\/3097983.3098024","relation":{},"subject":[],"published":{"date-parts":[[2017,8,13]]},"assertion":[{"value":"2017-08-13","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}