{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,21]],"date-time":"2026-02-21T19:47:46Z","timestamp":1771703266456,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,14]],"date-time":"2022-08-14T00:00:00Z","timestamp":1660435200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,14]]},"DOI":"10.1145\/3534678.3539160","type":"proceedings-article","created":{"date-parts":[[2022,8,12]],"date-time":"2022-08-12T19:06:12Z","timestamp":1660331172000},"page":"3168-3177","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["A\/B Testing Intuition Busters"],"prefix":"10.1145","author":[{"given":"Ron","family":"Kohavi","sequence":"first","affiliation":[{"name":"Kohavi, Los Altos, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alex","family":"Deng","sequence":"additional","affiliation":[{"name":"Airbnb Inc, Seattle, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lukas","family":"Vermeer","sequence":"additional","affiliation":[{"name":"Vista, Delft, Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,8,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10654-016-0149-3"},{"key":"e_1_3_2_1_2_1","volume-title":"June 2019","volume":"21","author":"Top","unstructured":"Top Challenges from the first Practical Online Controlled Experiments Summit. Gupta, Somit, et al. 1, June 2019, Vol. 21."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557139"},{"key":"e_1_3_2_1_4_1","first-page":"1367","volume-title":"IEEE International Conference on Big Data.","author":"Pitfalls","unstructured":"Pitfalls of long-term online controlled experiments. Dmitriev, Pavel, et al. Washington, DC : s.n., 2016. IEEE International Conference on Big Data. pp. 1367--1376."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1017\/9781108653985"},{"key":"e_1_3_2_1_6_1","first-page":"1427","volume-title":"Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD","author":"Dirty Dozen","year":"2017","unstructured":"A Dirty Dozen: Twelve Common Metric Interpretation Pitfalls in Online Controlled Experiments. Dmitriev, Pavel, et al. Halifax, NS, Canada : ACM, 2017. Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD 2017). pp. 1427--1436. 978--1--4503--4887--4."},{"key":"e_1_3_2_1_7_1","volume-title":"The Conference on Digital Experimentation (CODE@MIT).","author":"In Online Experimentation Platforms","unstructured":"Leaky Abstraction In Online Experimentation Platforms: A Conceptual Framework To Categorize Common Challenges. Kluck, Timo and Vermeer, Lukas. Boston, MA : s.n., 2015. The Conference on Digital Experimentation (CODE@MIT)."},{"key":"e_1_3_2_1_8_1","volume-title":"GuessTheTest. [Online]","author":"O'Malley Deborah","year":"2021","unstructured":"O'Malley, Deborah. Which design radically increased conversions 337%? GuessTheTest. [Online] December 16, 2021. https:\/\/guessthetest.com\/test\/which-design-radically-increased-conversions-337\/?referrer=Guessed."},{"key":"e_1_3_2_1_9_1","volume-title":"Open Science Collaboration. 6251","author":"Psychological Science","year":"2015","unstructured":"Estimating the Reproducibility of Psychological Science. Open Science Collaboration. 6251, 2015, Science, Vol. 349."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.7312\/clay19994"},{"key":"e_1_3_2_1_11_1","volume-title":"Steven.","author":"Dirty Dozen","year":"2008","unstructured":"A Dirty Dozen: Twelve P-Value Misconceptions. Goodman, Steven. 2008. Seminars in Hematology."},{"key":"e_1_3_2_1_12_1","volume-title":"What is a p-value anyway? 34 Stories to Help You Actually Understand Statistics. s.l. : Pearson","author":"Vickers Andrew J","year":"2009","unstructured":"Vickers, Andrew J. What is a p-value anyway? 34 Stories to Help You Actually Understand Statistics. s.l. : Pearson, 2009. 978-0321629302."},{"key":"e_1_3_2_1_13_1","volume-title":"Change the statistical significance setting","author":"Optimizely","year":"2022","unstructured":"Optimizely. Change the statistical significance setting. Optimizely Help Center. [Online] January 10, 2022. https:\/\/support.optimizely.com\/hc\/en-us\/articles\/4410289762189."},{"key":"e_1_3_2_1_14_1","volume-title":"The Most Powerful Way to Turn Clicks Into Customers. s.l. : Wiley","author":"Siroker Dan","year":"2013","unstructured":"Siroker, Dan and Koomen, Pete. A\/B Testing: The Most Powerful Way to Turn Clicks Into Customers. s.l. : Wiley, 2013. 1118536096."},{"key":"e_1_3_2_1_15_1","volume-title":"You Should Test That: Conversion Optimization for More Leads, Sales and Profit or The Art and Science of Optimized Marketing. s.l. : Sybex","author":"Goward Chris","year":"2012","unstructured":"Goward, Chris. You Should Test That: Conversion Optimization for More Leads, Sales and Profit or The Art and Science of Optimized Marketing. s.l. : Sybex, 2012. 1118301307."},{"key":"e_1_3_2_1_16_1","volume-title":"Causal Inference, and Social Science. [Online]","author":"Gelman Andrew","year":"2014","unstructured":"Gelman, Andrew. I didn't say that! Part 2. Statistical Modeling, Causal Inference, and Social Science. [Online] October 14, 2014. https:\/\/statmodeling.stat.columbia.edu\/2014\/10\/14\/didnt-say-part-2\/."},{"key":"e_1_3_2_1_17_1","unstructured":"The reproducibility of research and the misinterpretation of p-values. Colquhoun David. 4 2017 Royal Society Open Science."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1093\/jnci\/djh075"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-017-0189-z"},{"key":"e_1_3_2_1_22_1","volume-title":"Department of Biostatistics","author":"Goodman Steven","year":"2007","unstructured":"Goodman, Steven and Greenland, Sander. Assessing the unreliability of the medical literature: a response to \"Why most published research findings are false\". s.l. : Johns Hopkins University, Department of Biostatistics, 2007."},{"key":"e_1_3_2_1_23_1","volume-title":"Gabor Melli and Brendan Kitts. Third Workshop on Data Mining Case Studies and Practice Prize. 2009","author":"Kohavi Ron","year":"2009","unstructured":"Kohavi, Ron, Crook, Thomas and Longbotham, Roger. Online Experimentation at Microsoft. [ed.] Peter van der putten, Gabor Melli and Brendan Kitts. Third Workshop on Data Mining Case Studies and Practice Prize. 2009. http:\/\/www.appliedaisystems.com\/papers\/DMCS2009_Workshopproceedings4.pdf."},{"key":"e_1_3_2_1_24_1","volume-title":"Experimentation and Testing: A Primer. Occam's Razor by Avinash Kaushik. [Online]","author":"Kaushik Avinash","year":"2006","unstructured":"Kaushik, Avinash. Experimentation and Testing: A Primer. Occam's Razor by Avinash Kaushik. [Online] May 22, 2006."},{"key":"e_1_3_2_1_25_1","volume-title":"Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books","author":"Manzi Jim","year":"2012","unstructured":"Manzi, Jim. Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books, 2012. 978-0--465-02931-0."},{"key":"e_1_3_2_1_26_1","volume-title":"Stefan H. Experimentation Works: The Surprising Power of Business Experiments. s.l. : Harvard Business Review Press","author":"Thomke","year":"2020","unstructured":"Thomke, Stefan H. Experimentation Works: The Surprising Power of Business Experiments. s.l. : Harvard Business Review Press, 2020."},{"key":"e_1_3_2_1_27_1","volume-title":"Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press","author":"Moran Mike","year":"2007","unstructured":"Moran, Mike. Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press, 2007. 0132255960."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066X.45.12.1304"},{"key":"e_1_3_2_1_29_1","volume-title":"Statistical Rules of Thumb. s.l. : Wiley-Interscience","author":"van Belle Gerald","year":"2002","unstructured":"van Belle, Gerald. Statistical Rules of Thumb. s.l. : Wiley-Interscience, 2002. 0471402273."},{"key":"e_1_3_2_1_30_1","volume-title":"How to win in the Digital Experience Economy. Optimizely. [Online]","author":"Optimizely","year":"2021","unstructured":"Optimizely. How to win in the Digital Experience Economy. Optimizely. [Online] 2021. https:\/\/www.optimizely.com\/insights\/digital-experience-economy-report\/."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1177\/1745691614551642"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0045186"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1038\/nrn3475"},{"key":"e_1_3_2_1_35_1","volume-title":"KDD 2018: The 24th ACM Conference on Knowledge Discovery and Data Mining.","author":"Winner's Curse","unstructured":"Winner's Curse: Bias Estimation for Total Effects of Features in Online Controlled Experiments. Lee, Minyong R and Shen, Milan. London : ACM, 2018. KDD 2018: The 24th ACM Conference on Knowledge Discovery and Data Mining."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1086\/512821"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","first-page":"2743","DOI":"10.1145\/3447548.3467129","volume-title":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining.","author":"Tests Deng","year":"2021","unstructured":"On Post-Selection Inference in A\/B Tests. Deng, Alex, et al. 2021. Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining. pp. 2743--2752."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1097\/SLA.0000000000002908"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.annepidem.2012.02.007"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1198\/000313001300339897","volume":"55","author":"Data Analysis Hoenig","year":"2001","unstructured":"The Abuse of Power: The Pervasive Fallacy of Power Calculations for Data Analysis. Hoenig, John M and Heisey, Dennis M. 1, 2001, American Statistical Association, Vol. 55, pp. 19--24.","journal-title":"American Statistical Association"},{"key":"e_1_3_2_1_41_1","author":"Andr\u00e9 Quentin","year":"2021","unstructured":"Outlier exclusion procedures must be blind to the researcher's hypothesis. Andr\u00e9, Quentin. 2021, Journal of Experimental Psychology: General.","journal-title":"Journal of Experimental Psychology: General."},{"key":"e_1_3_2_1_42_1","first-page":"1517","volume-title":"KDD '17: Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining.","author":"Tests Johari","unstructured":"Peeking at A\/B Tests. Johari, Ramesh, et al. Halifax, NS, Canada : ACM, 2017. KDD '17: Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. pp. 1517--1525."},{"key":"e_1_3_2_1_43_1","first-page":"1359","volume-title":"Joseph P","author":"False-Positive Psychology","year":"2011","unstructured":"False-Positive Psychology: Undisclosed Flexibility in Data Collection and Analysis Allows Presenting Anything as Significant. Simmons, Joseph P, Nelson, Leif D and Simonsohn, Uri. 11, 2011, Psychological Science, Vol. 22, pp. 1359--1366."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1511\/2014.111.460"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0021524"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_1_47_1","volume-title":"KDD '19: The 25th SIGKDD International Conference on Knowledge Discovery and Data Mining.","author":"Online Controlled Experiments","unstructured":"Diagnosing Sample Ratio Mismatch in Online Controlled Experiments: A Taxonomy and Rules of Thumb for Practitioners. Fabijan, Aleksander, et al. Anchorage, Alaska, USA : ACM, 2019. KDD '19: The 25th SIGKDD International Conference on Knowledge Discovery and Data Mining."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.2307\/2686030"}],"event":{"name":"KDD '22: The 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Washington DC USA","acronym":"KDD '22","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539160","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3534678.3539160","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:58Z","timestamp":1750186978000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539160"}},"subtitle":["Common Misunderstandings in Online Controlled Experiments"],"short-title":[],"issued":{"date-parts":[[2022,8,14]]},"references-count":48,"alternative-id":["10.1145\/3534678.3539160","10.1145\/3534678"],"URL":"https:\/\/doi.org\/10.1145\/3534678.3539160","relation":{},"subject":[],"published":{"date-parts":[[2022,8,14]]},"assertion":[{"value":"2022-08-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}