{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,31]],"date-time":"2026-03-31T12:02:10Z","timestamp":1774958530517,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":57,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,24]],"date-time":"2024-08-24T00:00:00Z","timestamp":1724457600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,25]]},"DOI":"10.1145\/3637528.3671631","type":"proceedings-article","created":{"date-parts":[[2024,8,25]],"date-time":"2024-08-25T04:55:12Z","timestamp":1724561712000},"page":"5240-5250","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["False Positives in A\/B Tests"],"prefix":"10.1145","author":[{"given":"Ron","family":"Kohavi","sequence":"first","affiliation":[{"name":"Kohavi, Los Altos, CA, USA"}]},{"given":"Nanyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Expedia Group, San Francisco, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,8,24]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1017\/9781108653985"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7502-7_891-2"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/11891.001.0001"},{"key":"e_1_3_2_2_4_1","volume-title":"Stefan H. Experimentation Works: The Surprising Power of Business Experiments. s.l.: Harvard Business Review Press","author":"Thomke","year":"2020","unstructured":"Thomke, Stefan H. Experimentation Works: The Surprising Power of Business Experiments. s.l.: Harvard Business Review Press, 2020."},{"key":"e_1_3_2_2_5_1","volume-title":"Statistical Methods in Online A\/B Testing: Statistics for data-driven business decisions and risk management in e-commerce. s.l.: Independently published","author":"Georgiev Georgi Zdravkov","year":"2019","unstructured":"Georgiev, Georgi Zdravkov. Statistical Methods in Online A\/B Testing: Statistics for data-driven business decisions and risk management in e-commerce. s.l.: Independently published, 2019. 978--1694079725."},{"key":"e_1_3_2_2_6_1","volume-title":"Design and Analysis of Experiments","author":"Montgomery Douglas C","year":"2019","unstructured":"Montgomery, Douglas C. Design and Analysis of Experiments. 10th edition. s.l.: Wiley, 2019.","edition":"10"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139025751"},{"key":"e_1_3_2_2_8_1","volume-title":"2nd. s.l.: John Wiley & Sons","author":"Box George E.P.","year":"2005","unstructured":"Box, George E.P., Hunter, J Stuart and Hunter, William G. Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l.: John Wiley & Sons, Inc, 2005. 0471718130."},{"key":"e_1_3_2_2_9_1","volume-title":"Donald P. Field Experiments: Design, Analysis, and Interpretation. s.l.: W","author":"Gerber Alan S","year":"2012","unstructured":"Gerber, Alan S and Green, Donald P. Field Experiments: Design, Analysis, and Interpretation. s.l.: W. W. Norton & Company, 2012. 978-0393979954."},{"key":"e_1_3_2_2_10_1","volume-title":"Nicholas, et al. 2, s.l.: The American Statistician","author":"Online Controlled Experiments","year":"2023","unstructured":"Statistical Challenges in Online Controlled Experiments: A Review of A\/B Testing Methodology. Larsen, Nicholas, et al. 2, s.l.: The American Statistician, 2023, Vol. 78."},{"key":"e_1_3_2_2_11_1","first-page":"74","volume-title":"Stefan.","author":"Online Experiments","year":"2017","unstructured":"The Surprising Power of Online Experiments: Getting the most out of A\/B and other controlled tests. Kohavi, Ron and Thomke, Stefan. Sept-October, 2017, Harvard Business Review, pp. 74--92."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_3_2_2_13_1","volume-title":"June 2019","volume":"21","author":"Top","unstructured":"Top Challenges from the first Practical Online Controlled Experiments Summit. Gupta, Somit, et al. 1, June 2019, Vol. 21."},{"key":"e_1_3_2_2_14_1","volume-title":"USA: ACM","author":"Testing Intuition Busters","year":"2022","unstructured":"A\/B Testing Intuition Busters: Common Misunderstandings in Online Controlled Experiments. Kohavi, Ron, Deng, Alex and Vermeer, Lukas. Washington DC, USA: ACM, New York, NY, USA, 2022. Proceedings of the 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD '22)."},{"key":"e_1_3_2_2_15_1","unstructured":"The reproducibility of research and the misinterpretation of p-values. Colquhoun David. 4 2017 Royal Society Open Science."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1093\/jnci\/djh075"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623341"},{"key":"e_1_3_2_2_19_1","volume-title":"s.l.: American Journal of Sociology","author":"Statistical Learning Gelman","year":"2011","unstructured":"Causality and Statistical Learning. Gelman, Andrew. 3, s.l.: American Journal of Sociology, 2011, Vol. 117."},{"key":"e_1_3_2_2_20_1","volume-title":"Open Science Collaboration. 6251","author":"Psychological Science","year":"2015","unstructured":"Estimating the Reproducibility of Psychological Science. Open Science Collaboration. 6251, 2015, Science, Vol. 349."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.74.5.1252"},{"key":"e_1_3_2_2_22_1","volume-title":"Replicability Report No. 1: Is Ego-Depletion a Replicable Effect? replicability-Index. [Online]","author":"Schimmack Ulrich","year":"2016","unstructured":"Schimmack, Ulrich. Replicability Report No. 1: Is Ego-Depletion a Replicable Effect? replicability-Index. [Online] April 18, 2016. https:\/\/replicationindex.com\/2016\/04\/18\/is-replicability-report-ego-depletionreplicability-report-of-165-ego-depletion-articles\/."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1177\/0956797621989733"},{"key":"e_1_3_2_2_24_1","first-page":"638","volume-title":"Psychological Bulletin","author":"Null Results Rosenthal","unstructured":"The \"File Drawer Problem\" and Tolerance for Null Results. Rosenthal, Robert. 3, 1979, Psychological Bulletin, Vol. 86, pp. 638--641."},{"key":"e_1_3_2_2_25_1","volume-title":"Steven.","author":"Dirty Dozen","year":"2008","unstructured":"A Dirty Dozen: Twelve P-Value Misconceptions. Goodman, Steven. 2008. Seminars in Hematology."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10654-016-0149-3"},{"key":"e_1_3_2_2_27_1","volume-title":"What is a p-value anyway? 34 Stories to Help You Actually Understand Statistics. s.l.: Pearson","author":"Vickers Andrew J","year":"2009","unstructured":"Vickers, Andrew J. What is a p-value anyway? 34 Stories to Help You Actually Understand Statistics. s.l.: Pearson, 2009. 978-0321629302."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"crossref","unstructured":"Power or Alpha? The Better Way of Decreasing the False Discovery Rate. Barto? Franti?ek and Maier Maximilian. Nov 08 2022 Meta-Psychology Vol. 6.","DOI":"10.15626\/MP.2020.2460"},{"key":"e_1_3_2_2_29_1","volume-title":"Franti?ek and Schimmack, Ulrich.","year":"2022","unstructured":"Z-curve 2.0: Estimating Replication Rates and Discovery Rates. Barto?, Franti?ek and Schimmack, Ulrich. 2022, Meta-Psychology, Vol. 6."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-017-0189-z"},{"key":"e_1_3_2_2_31_1","volume-title":"Confidence intervals and improvement intervals. Optimizely. [Online]","author":"Optimizely","year":"2023","unstructured":"Optimizely. Confidence intervals and improvement intervals. Optimizely. [Online] 2023. https:\/\/support.optimizely.com\/hc\/en-us\/articles\/4410283895821-Confidence-intervals-and-improvement-intervals."},{"key":"e_1_3_2_2_32_1","volume-title":"One-tailed vs Two-tailed Tests of Significance in A\/B Testing. Analytics Toolkit. [Online]","author":"Georgiev Georgi","year":"2018","unstructured":"Georgiev, Georgi. One-tailed vs Two-tailed Tests of Significance in A\/B Testing. Analytics Toolkit. [Online] August 8, 2018. https:\/\/blog.analytics-toolkit.com\/2017\/one-tailed-two-tailed-tests-significance-ab-testing\/."},{"key":"e_1_3_2_2_33_1","volume-title":"Raising the bar by lowering the bound. Booking.ai. [Online]","author":"Skotara Nils","year":"2023","unstructured":"Skotara, Nils. Raising the bar by lowering the bound. Booking.ai. [Online] Nov 1, 2023. https:\/\/booking.ai\/raising-the-bar-by-lowering-the-bound-3b12d3bd43a3."},{"key":"e_1_3_2_2_34_1","volume-title":"Evolution of Experimentation","author":"Gabster Elizabeth","year":"2023","unstructured":"Gabster, Elizabeth, et al. Evolution of Experimentation. 2023."},{"key":"e_1_3_2_2_35_1","volume-title":"Statistics Done Wrong: The Woefully Complete Guide. s.l.: No Starch Press","author":"Reinhart Alex","year":"2015","unstructured":"Reinhart, Alex. Statistics Done Wrong: The Woefully Complete Guide. s.l.: No Starch Press, 2015. 978--1593276201."},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1177\/1745691614551642"},{"key":"e_1_3_2_2_37_1","volume-title":"Why positive A\/B test results should always be given a haircut","author":"Kohavi Ron","year":"2023","unstructured":"Kohavi, Ron. Why positive A\/B test results should always be given a haircut. Dec 3, 2023."},{"key":"e_1_3_2_2_38_1","volume-title":"Milan. London: ACM, 2018. KDD 2018: The 24th ACM Conference on Knowledge Discovery and Data Mining.","author":"Winner's Curse","unstructured":"Winner's Curse: Bias Estimation for Total Effects of Features in Online Controlled Experiments. Lee, Minyong R and Shen, Milan. London: ACM, 2018. KDD 2018: The 24th ACM Conference on Knowledge Discovery and Data Mining."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1086\/512821"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"crossref","first-page":"2743","DOI":"10.1145\/3447548.3467129","volume-title":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining.","author":"Tests Deng","year":"2021","unstructured":"On Post-Selection Inference in A\/B Tests. Deng, Alex, et al. 2021. Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining. pp. 2743--2752."},{"key":"e_1_3_2_2_41_1","volume-title":"Gabor Melli and Brendan Kitts. Third Workshop on Data Mining Case Studies and Practice Prize. 2009","author":"Kohavi Ron","year":"2009","unstructured":"Kohavi, Ron, Crook, Thomas and Longbotham, Roger. Online Experimentation at Microsoft. [ed.] Peter van der putten, Gabor Melli and Brendan Kitts. Third Workshop on Data Mining Case Studies and Practice Prize. 2009. http:\/\/www.appliedaisystems.com\/papers\/DMCS2009_Workshopproceedings4.pdf."},{"key":"e_1_3_2_2_42_1","volume-title":"Fisher's method. Wikipedia. [Online]","year":"2023","unstructured":"Wikipedia contributors. Fisher's method. Wikipedia. [Online] Dec 2023. http:\/\/en.wikipedia.org\/wiki\/Fisher%27s_method."},{"key":"e_1_3_2_2_43_1","volume-title":"Meta Analysis Spreadsheet","author":"Kohavi Ron","year":"2021","unstructured":"Kohavi, Ron. Meta Analysis Spreadsheet. 2021."},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.1995.10476125"},{"key":"e_1_3_2_2_45_1","volume":"128","author":"Fat Tails Azevedo","year":"2020","unstructured":"A\/B Testing with Fat Tails. Azevedo, Eduardo M., et al. 12, 2020, Journal of Political Economy, Vol. 128.","journal-title":"Journal of Political Economy"},{"key":"e_1_3_2_2_46_1","volume-title":"What Can Be Learned From 1,001 A\/B Tests?","author":"Georgiev Georgi","year":"2022","unstructured":"Georgiev, Georgi. What Can Be Learned From 1,001 A\/B Tests? Oct 17, 2022."},{"key":"e_1_3_2_2_47_1","volume-title":"2nd.","author":"Casella George","year":"2002","unstructured":"Casella, George and Berger, Roger. Statistical Inference. 2nd. 2002."},{"key":"e_1_3_2_2_48_1","volume-title":"Korea: International World Wide Web Conference (IW3C2)","author":"Validation Deng","year":"2014","unstructured":"Statistical Inference in Two-Stage Online Controlled Experiments with Treatment Selection and Validation. Deng, Alex, Li, Tianxi and Guo, Yu. Seoul, Korea: International World Wide Web Conference (IW3C2), 2014."},{"key":"e_1_3_2_2_49_1","volume-title":"The alpha spending function approach to interim data analyses. [book auth.] P.F. Thall. Recent Advances in Clinical Trial Design and Analysis. s.l.: Springer","author":"DeMets David L","year":"1995","unstructured":"DeMets, David L. and Lan, Gordon. The alpha spending function approach to interim data analyses. [book auth.] P.F. Thall. Recent Advances in Clinical Trial Design and Analysis. s.l.: Springer, 1995."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.2307\/2336502"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/64.2.191"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.2307\/2530245"},{"key":"e_1_3_2_2_53_1","volume-title":"Medium.","author":"Arguelles Carlos","year":"2021","unstructured":"Arguelles, Carlos. The Paradigm Shifts with Different Dev:Test Ratios. Medium. Aug 31, 2021."},{"key":"e_1_3_2_2_54_1","volume-title":"The Lean Startup: How Today's Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses. s.l.: Crown Business","author":"Ries Eric","year":"2011","unstructured":"Ries, Eric. The Lean Startup: How Today's Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses. s.l.: Crown Business, 2011. 978-0307887894."},{"key":"e_1_3_2_2_55_1","volume-title":"Network World.","author":"Patrizio Andy","year":"2014","unstructured":"Patrizio, Andy. Would Microsoft really cut its QA department? Network World. July 15, 2014."},{"key":"e_1_3_2_2_56_1","volume-title":"The Power of Experimentation: A\/B testing for startups and low traffic websites","author":"Longden Jonny","year":"2023","unstructured":"Longden, Jonny. The Power of Experimentation: A\/B testing for startups and low traffic websites. February 2023."},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.7312\/clay19994"}],"event":{"name":"KDD '24: The 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Barcelona Spain","acronym":"KDD '24","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3637528.3671631","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3637528.3671631","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:05:59Z","timestamp":1750291559000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3637528.3671631"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,24]]},"references-count":57,"alternative-id":["10.1145\/3637528.3671631","10.1145\/3637528"],"URL":"https:\/\/doi.org\/10.1145\/3637528.3671631","relation":{},"subject":[],"published":{"date-parts":[[2024,8,24]]},"assertion":[{"value":"2024-08-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}