{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,28]],"date-time":"2026-01-28T11:57:25Z","timestamp":1769601445412,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2012,8,12]],"date-time":"2012-08-12T00:00:00Z","timestamp":1344729600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2012,8,12]]},"DOI":"10.1145\/2339530.2339653","type":"proceedings-article","created":{"date-parts":[[2012,8,14]],"date-time":"2012-08-14T12:18:56Z","timestamp":1344946736000},"page":"786-794","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":136,"title":["Trustworthy online controlled experiments"],"prefix":"10.1145","author":[{"given":"Ron","family":"Kohavi","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Alex","family":"Deng","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Brian","family":"Frasca","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Roger","family":"Longbotham","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Toby","family":"Walker","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Ya","family":"Xu","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]}],"member":"320","published-online":{"date-parts":[[2012,8,12]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"CA : s.n.","author":"Kohavi Ron","year":"2004","unstructured":"Kohavi , Ron and Round , Matt . Front Line Internet Analytics at Amazon.com. {ed.} Jim Sterne. Santa Barbara , CA : s.n. , 2004 . http:\/\/ai.stanford.edu\/~ronnyk\/emetricsAmazon.pdf. Kohavi, Ron and Round, Matt. Front Line Internet Analytics at Amazon.com. {ed.} Jim Sterne. Santa Barbara, CA : s.n., 2004. http:\/\/ai.stanford.edu\/~ronnyk\/emetricsAmazon.pdf."},{"key":"e_1_3_2_2_2_1","volume-title":"Third Workshop on Data Mining Case Studies and Practice Prize.","author":"Kohavi Ron","year":"2009","unstructured":"Kohavi , Ron , Crook , Thomas and Longbotham , Roger . Online Experimentation at Microsoft . Third Workshop on Data Mining Case Studies and Practice Prize. 2009 . http:\/\/exp-platform.com\/expMicrosoft.aspx. Kohavi, Ron, Crook, Thomas and Longbotham, Roger. Online Experimentation at Microsoft. Third Workshop on Data Mining Case Studies and Practice Prize. 2009. http:\/\/exp-platform.com\/expMicrosoft.aspx."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_2_4_1","volume-title":"Competing on Analytics at the Highest Level. {Online}","author":"Patil DJ","year":"2010","unstructured":"Patil , DJ , Competing on Analytics at the Highest Level. {Online} Apr 9, 2010 . http:\/\/www.svforum.org\/index.cfm?fuseaction=Page.ViewPage&PageID=997. Patil, DJ, et al. Competing on Analytics at the Highest Level. {Online} Apr 9, 2010. http:\/\/www.svforum.org\/index.cfm?fuseaction=Page.ViewPage&PageID=997."},{"key":"e_1_3_2_2_5_1","volume-title":"Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books","author":"Manzi Jim","year":"2012","unstructured":"Manzi , Jim . Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books , 2012 . 978-0--465-02931-0. Manzi, Jim. Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books, 2012. 978-0--465-02931-0."},{"key":"e_1_3_2_2_6_1","volume-title":"Experimentation and Testing: A Primer. Occam's Razor. {Online}","author":"Kaushik Avinash","year":"2006","unstructured":"Kaushik , Avinash . Experimentation and Testing: A Primer. Occam's Razor. {Online} May 22, 2006 . http:\/\/www.kaushik.net\/avinash\/2006\/05\/experimentation-and-testing-a-primer.html. Kaushik, Avinash. Experimentation and Testing: A Primer. Occam's Razor. {Online} May 22, 2006. http:\/\/www.kaushik.net\/avinash\/2006\/05\/experimentation-and-testing-a-primer.html."},{"key":"e_1_3_2_2_7_1","volume-title":"Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press","author":"Moran Mike","year":"2007","unstructured":"Moran , Mike . Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press , 2007 . 0132255960. Moran, Mike. Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press, 2007. 0132255960."},{"key":"e_1_3_2_2_8_1","volume-title":"Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online}","author":"Action","year":"2008","unstructured":"Multivariate Testing in Action : Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online} December 23, 2008 . http:\/\/www.biznology.com\/2008\/12\/multivariate_testing_in_action\/. Multivariate Testing in Action: Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online} December 23, 2008. http:\/\/www.biznology.com\/2008\/12\/multivariate_testing_in_action\/."},{"key":"e_1_3_2_2_9_1","volume-title":"Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley & Sons","author":"Mason Robert L","year":"1989","unstructured":"Mason , Robert L , Gunst , Richard F and Hess , James L . Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley & Sons , 1989 . 047185364X. Mason, Robert L, Gunst, Richard F and Hess, James L. Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley & Sons, 1989. 047185364X."},{"key":"e_1_3_2_2_10_1","volume-title":"2nd. s.l. : John Wiley & Sons","author":"Box George E.P.","year":"2005","unstructured":"Box , George E.P. , Hunter , J Stuart and Hunter , William G . Statistics for Experimenters: Design, Innovation, and Discovery . 2nd. s.l. : John Wiley & Sons , Inc , 2005 . 0471718130. Box, George E.P., Hunter, J Stuart and Hunter, William G. Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l. : John Wiley & Sons, Inc, 2005. 0471718130."},{"key":"e_1_3_2_2_11_1","volume-title":"2nd. s.l.","author":"Keppel Geoffrey","year":"1992","unstructured":"Keppel , Geoffrey , Saufley , William H and Tokunaga , Howard . Introduction to Design and Analysis . 2nd. s.l. : W.H. Freeman and Company , 1992 . Keppel, Geoffrey, Saufley, William H and Tokunaga, Howard. Introduction to Design and Analysis. 2nd. s.l. : W.H. Freeman and Company, 1992."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281295"},{"key":"e_1_3_2_2_13_1","volume-title":"Web Analytics Demystified: A Marketer's Guide to Understanding How Your Web Site Affects Your Business. s.l. : Celilo Group Media and CafePress","author":"Peterson Eric T","year":"2004","unstructured":"Peterson , Eric T . Web Analytics Demystified: A Marketer's Guide to Understanding How Your Web Site Affects Your Business. s.l. : Celilo Group Media and CafePress , 2004 . 0974358428. Peterson, Eric T. Web Analytics Demystified: A Marketer's Guide to Understanding How Your Web Site Affects Your Business. s.l. : Celilo Group Media and CafePress, 2004. 0974358428."},{"key":"e_1_3_2_2_14_1","first-page":"3500811","volume-title":"How to Improve A\/B Testing. ClickZ Network. {Online}","author":"Eisenberg Bryan","year":"2005","unstructured":"Eisenberg , Bryan . How to Improve A\/B Testing. ClickZ Network. {Online} April 29, 2005 . http:\/\/www.clickz.com\/showPage.html?page= 3500811 . Eisenberg, Bryan. How to Improve A\/B Testing. ClickZ Network. {Online} April 29, 2005. http:\/\/www.clickz.com\/showPage.html?page=3500811."},{"key":"e_1_3_2_2_15_1","volume-title":"A Primer on A\/B Testing. s.l. : Forrester Research","author":"Chatham Bob","year":"2004","unstructured":"Chatham , Bob , Temkin , Bruce D and Amato , Michelle . A Primer on A\/B Testing. s.l. : Forrester Research , 2004 . Chatham, Bob, Temkin, Bruce D and Amato, Michelle. A Primer on A\/B Testing. s.l. : Forrester Research, 2004."},{"key":"e_1_3_2_2_16_1","volume-title":"Conversion\/Testing. {Online}","author":"Miller Scott","year":"2007","unstructured":"Miller , Scott . How to Design a Split Test. Web Marketing Today , Conversion\/Testing. {Online} Jan 18, 2007 . http:\/\/www.wilsonweb.com\/conversion\/. Miller, Scott. How to Design a Split Test. Web Marketing Today, Conversion\/Testing. {Online} Jan 18, 2007. http:\/\/www.wilsonweb.com\/conversion\/."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_2_18_1","volume-title":"Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley & Sons","author":"Roy Ranjit K","year":"2001","unstructured":"Roy , Ranjit K . Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley & Sons , Inc , 2001 . 0--471--36101--1. Roy, Ranjit K. Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley & Sons, Inc, 2001. 0--471--36101--1."},{"key":"e_1_3_2_2_19_1","volume-title":"Too Soon. Future Now. {Online}","author":"vonTivadar John","year":"2006","unstructured":"Quarto- vonTivadar , John . AB Testing: Too Little , Too Soon. Future Now. {Online} 2006 . http:\/\/www.futurenowinc.com\/abtesting.pdf. Quarto-vonTivadar, John. AB Testing: Too Little, Too Soon. Future Now. {Online} 2006. http:\/\/www.futurenowinc.com\/abtesting.pdf."},{"key":"e_1_3_2_2_20_1","volume-title":"The Balanced Scorecard: Translating Strategy into Action. s.l. : Harvard Business School Press","author":"Kaplan Robert S","year":"1996","unstructured":"Kaplan , Robert S and Norton , David P. The Balanced Scorecard: Translating Strategy into Action. s.l. : Harvard Business School Press , 1996 . 0875846513. Kaplan, Robert S and Norton, David P. The Balanced Scorecard: Translating Strategy into Action. s.l. : Harvard Business School Press, 1996. 0875846513."},{"key":"e_1_3_2_2_21_1","unstructured":"Cheshire Tom. Test. Test. Test: How wooga turned the games business into a science. Wired Magazine UK http:\/\/www.wired.co.uk\/magazine\/archive\/2012\/01\/features\/test-test-test?page=all. January 5 2012.  Cheshire Tom. Test. Test. Test: How wooga turned the games business into a science. Wired Magazine UK http:\/\/www.wired.co.uk\/magazine\/archive\/2012\/01\/features\/test-test-test?page=all. January 5 2012."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557139"},{"key":"e_1_3_2_2_23_1","volume-title":"WA : s.n.","author":"Kohavi Ron","year":"2010","unstructured":"Kohavi , Ron , et al. Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss. Redmond , WA : s.n. , 2010 . Microsoft White pape, http:\/\/www.exp-platform.com\/Pages\/TrackingClicksSubmits.aspx. Kohavi, Ron, et al. Tracking Users' Clicks and Submits: Tradeoffs between User Experience and Data Loss. Redmond, WA : s.n., 2010. Microsoft White pape, http:\/\/www.exp-platform.com\/Pages\/TrackingClicksSubmits.aspx."},{"key":"e_1_3_2_2_24_1","unstructured":"Hawthorne effect. Wikipedia. {Online} 2007. http:\/\/en.wikipedia.org\/wiki\/Hawthorne_experiments.  Hawthorne effect. Wikipedia. {Online} 2007. http:\/\/en.wikipedia.org\/wiki\/Hawthorne_experiments."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1037\/1089-2680.2.2.175"},{"key":"e_1_3_2_2_26_1","unstructured":"Rosset Saharon and Borodovsky Slava. A\/B Testing Using the Negative Binomial Distribution in an Internet Search Application Submitted. {Online} 2012. http:\/\/www.tau.ac.il\/~saharon\/papers\/AB%20testing%20with%20NB%20distribution%20-%20revision.pdf.  Rosset Saharon and Borodovsky Slava. A\/B Testing Using the Negative Binomial Distribution in an Internet Search Application Submitted. {Online} 2012. http:\/\/www.tau.ac.il\/~saharon\/papers\/AB%20testing%20with%20NB%20distribution%20-%20revision.pdf."},{"key":"e_1_3_2_2_27_1","volume-title":"2nd. s.l.","author":"Weiss","year":"1997","unstructured":"Weiss , Carol H. Evaluation: Methods for Studying Programs and Policies . 2nd. s.l. : Prentice Hall , 1997 . 0--13--309725-0. Weiss, Carol H. Evaluation: Methods for Studying Programs and Policies. 2nd. s.l. : Prentice Hall, 1997. 0--13--309725-0."}],"event":{"name":"KDD '12: The 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","location":"Beijing China","acronym":"KDD '12","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 18th ACM SIGKDD international conference on Knowledge discovery and data mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2339530.2339653","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2339530.2339653","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T09:34:13Z","timestamp":1750239253000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2339530.2339653"}},"subtitle":["five puzzling outcomes explained"],"short-title":[],"issued":{"date-parts":[[2012,8,12]]},"references-count":27,"alternative-id":["10.1145\/2339530.2339653","10.1145\/2339530"],"URL":"https:\/\/doi.org\/10.1145\/2339530.2339653","relation":{},"subject":[],"published":{"date-parts":[[2012,8,12]]},"assertion":[{"value":"2012-08-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}