{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T22:17:32Z","timestamp":1768342652107,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,8,11]],"date-time":"2013-08-11T00:00:00Z","timestamp":1376179200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,8,11]]},"DOI":"10.1145\/2487575.2488217","type":"proceedings-article","created":{"date-parts":[[2013,8,13]],"date-time":"2013-08-13T12:31:21Z","timestamp":1376397081000},"page":"1168-1176","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":264,"title":["Online controlled experiments at large scale"],"prefix":"10.1145","author":[{"given":"Ron","family":"Kohavi","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Alex","family":"Deng","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Brian","family":"Frasca","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Toby","family":"Walker","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Ya","family":"Xu","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]},{"given":"Nils","family":"Pohlmann","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}]}],"member":"320","published-online":{"date-parts":[[2013,8,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"CA : s.n.","author":"Kohavi Ron","year":"2004","unstructured":"Kohavi , Ron and Round , Matt . Front Line Internet Analytics at Amazon.com. {ed.} Jim Sterne. Santa Barbara , CA : s.n. , 2004 . http:\/\/ai.stanford.edu\/~ronnyk\/emetricsAmazon.pdf. Kohavi, Ron and Round, Matt. Front Line Internet Analytics at Amazon.com. {ed.} Jim Sterne. Santa Barbara, CA : s.n., 2004. http:\/\/ai.stanford.edu\/~ronnyk\/emetricsAmazon.pdf."},{"key":"e_1_3_2_1_2_1","volume-title":"Design for Continuous Experimentation: Talk and Slides. {Online}","author":"McKinley Dan","year":"2012","unstructured":"McKinley , Dan . Design for Continuous Experimentation: Talk and Slides. {Online} Dec 22, 2012 . http:\/\/mcfunley.com\/design-for-continuous-experimentation. McKinley, Dan. Design for Continuous Experimentation: Talk and Slides. {Online} Dec 22, 2012. http:\/\/mcfunley.com\/design-for-continuous-experimentation."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_2_1_4_1","volume-title":"Multivariate Testing in Action: Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online}","author":"Moran Mike","year":"2008","unstructured":"Moran , Mike . Multivariate Testing in Action: Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online} December 2008 . www.biznology.com\/2008\/12\/multivariate_testing_in_action\/. Moran, Mike. Multivariate Testing in Action: Quicken Loan's Regis Hadiaris on multivariate testing. Biznology Blog by Mike Moran. {Online} December 2008. www.biznology.com\/2008\/12\/multivariate_testing_in_action\/."},{"key":"e_1_3_2_1_5_1","volume-title":"Third Workshop on Data Mining Case Studies and Practice Prize.","author":"Kohavi Ron","year":"2009","unstructured":"Kohavi , Ron , Crook , Thomas and Longbotham , Roger . Online Experimentation at Microsoft . Third Workshop on Data Mining Case Studies and Practice Prize. 2009 . http:\/\/exp-platform.com\/expMicrosoft.aspx. Kohavi, Ron, Crook, Thomas and Longbotham, Roger. Online Experimentation at Microsoft. Third Workshop on Data Mining Case Studies and Practice Prize. 2009. http:\/\/exp-platform.com\/expMicrosoft.aspx."},{"key":"e_1_3_2_1_6_1","volume-title":"Netflix Recommendations: Beyond the 5 stars. {Online}","author":"Amatriain Xavier","year":"2012","unstructured":"Amatriain , Xavier and Basilico , Justin . Netflix Recommendations: Beyond the 5 stars. {Online} April 2012 . http:\/\/techblog.netflix.com\/2012\/04\/netflix-recommendations-beyond-5-stars.html. Amatriain, Xavier and Basilico , Justin. Netflix Recommendations: Beyond the 5 stars. {Online} April 2012. http:\/\/techblog.netflix.com\/2012\/04\/netflix-recommendations-beyond-5-stars.html."},{"key":"e_1_3_2_1_7_1","volume-title":"Website conversion rate optimization with A\/B and multivariate testing. s.l. : New Riders","author":"McFarland Colin","year":"2012","unstructured":"McFarland , Colin . Experiment! : Website conversion rate optimization with A\/B and multivariate testing. s.l. : New Riders , 2012 . 978-0321834607. McFarland, Colin. Experiment!: Website conversion rate optimization with A\/B and multivariate testing. s.l. : New Riders, 2012. 978-0321834607."},{"key":"e_1_3_2_1_8_1","volume-title":"Recommendations and Discovery at StumbleUpon. {Online}","author":"Kolar Sumanth","year":"2012","unstructured":"Kolar , Sumanth . Recommendations and Discovery at StumbleUpon. {Online} Sept 2012 . www.slideshare.net\/sumanthkolar\/recsys-2012-sumanth-14260370. Kolar, Sumanth. Recommendations and Discovery at StumbleUpon. {Online} Sept 2012. www.slideshare.net\/sumanthkolar\/recsys-2012-sumanth-14260370."},{"key":"e_1_3_2_1_9_1","volume-title":"Zynga: What is Zynga's core competency? Quora. {Online}","author":"Smietana Brandon","year":"2010","unstructured":"Smietana , Brandon . Zynga: What is Zynga's core competency? Quora. {Online} Sept 2010 . http:\/\/www.quora.com\/Zynga\/What-is-Zyngas-core-competency\/answer\/Brandon-Smietana. Smietana, Brandon. Zynga: What is Zynga's core competency? Quora. {Online} Sept 2010. http:\/\/www.quora.com\/Zynga\/What-is-Zyngas-core-competency\/answer\/Brandon-Smietana."},{"key":"e_1_3_2_1_10_1","volume-title":"The Four Steps to the Epiphany: Successful Strategies for Products that Win . s.l. : Cafepress.com","author":"Blank Steven Gary","year":"2005","unstructured":"Blank , Steven Gary . The Four Steps to the Epiphany: Successful Strategies for Products that Win . s.l. : Cafepress.com , 2005 . 978-0976470700. Blank, Steven Gary. The Four Steps to the Epiphany: Successful Strategies for Products that Win . s.l. : Cafepress.com, 2005. 978-0976470700."},{"key":"e_1_3_2_1_11_1","volume-title":"The Lean Startup: How Today's Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses . s.l. : Crown Business","author":"Ries Eric","year":"2011","unstructured":"Ries , Eric . The Lean Startup: How Today's Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses . s.l. : Crown Business , 2011 . 978-0307887894. Ries, Eric. The Lean Startup: How Today's Entrepreneurs Use Continuous Innovation to Create Radically Successful Businesses . s.l. : Crown Business, 2011. 978-0307887894."},{"key":"e_1_3_2_1_12_1","unstructured":"Wikipedia. Lean Startup. {Online} 2013. http:\/\/en.wikipedia.org\/wiki\/Lean_Startup.  Wikipedia. Lean Startup. {Online} 2013. http:\/\/en.wikipedia.org\/wiki\/Lean_Startup."},{"key":"e_1_3_2_1_13_1","volume-title":"One Strategy: Organization, Planning, and Decision Making . s.l. : Wiley","author":"Sinofsky Steven","year":"2009","unstructured":"Sinofsky , Steven . One Strategy: Organization, Planning, and Decision Making . s.l. : Wiley , 2009 . 978-0470560457 . Sinofsky, Steven. One Strategy: Organization, Planning, and Decision Making . s.l. : Wiley, 2009. 978-0470560457 ."},{"key":"e_1_3_2_1_14_1","volume-title":"comScore Releases","author":"Score","year":"2013","unstructured":"com Score . comScore Releases January 2013 U.S. Search Engine Rankings . {Online} Feb 13, 2013. http:\/\/www.comscore.com\/Insights\/Press_Releases\/2013\/2\/comScore_Releases_January_2013_U.S._Search_Engine_Rankings. comScore. comScore Releases January 2013 U.S. Search Engine Rankings. {Online} Feb 13, 2013. http:\/\/www.comscore.com\/Insights\/Press_Releases\/2013\/2\/comScore_Releases_January_2013_U.S._Search_Engine_Rankings."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00778-012-0280-z"},{"key":"e_1_3_2_1_16_1","volume-title":"Microsoft Second Quarter 2013 Earnings Calls Transcript. Microsoft Investor Relations. {Online}","author":"Klein Peter","year":"2013","unstructured":"Klein , Peter and Suh , Chris . Microsoft Second Quarter 2013 Earnings Calls Transcript. Microsoft Investor Relations. {Online} Jan 24, 2013 . http:\/\/www.microsoft.com\/global\/Investor\/RenderingAssets\/Downloads\/FY13\/Q2\/Microsoft_Q2_2013_PreparedRemarks.docx. Klein, Peter and Suh, Chris. Microsoft Second Quarter 2013 Earnings Calls Transcript. Microsoft Investor Relations. {Online} Jan 24, 2013. http:\/\/www.microsoft.com\/global\/Investor\/RenderingAssets\/Downloads\/FY13\/Q2\/Microsoft_Q2_2013_PreparedRemarks.docx."},{"key":"e_1_3_2_1_17_1","volume-title":"Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books","author":"Manzi Jim","year":"2012","unstructured":"Manzi , Jim . Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books , 2012 . 978-0-465-02931-0. Manzi, Jim. Uncontrolled: The Surprising Payoff of Trial-and-Error for Business, Politics, and Society. s.l. : Basic Books, 2012. 978-0-465-02931-0."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_19_1","volume-title":"How to Improve A\/B Testing. ClickZ Network. {Online}","author":"Eisenberg Bryan","year":"2005","unstructured":"Eisenberg , Bryan . How to Improve A\/B Testing. ClickZ Network. {Online} April 29, 2005 . www.clickz.com\/clickz\/column\/1717234\/how-improve-a-b-testing. Eisenberg, Bryan. How to Improve A\/B Testing. ClickZ Network. {Online} April 29, 2005. www.clickz.com\/clickz\/column\/1717234\/how-improve-a-b-testing."},{"key":"e_1_3_2_1_20_1","volume-title":"2nd. s.l. : John Wiley & Sons","author":"Box George E.P.","year":"2005","unstructured":"Box , George E.P. , Hunter , J Stuart and Hunter , William G . Statistics for Experimenters: Design, Innovation, and Discovery . 2nd. s.l. : John Wiley & Sons , Inc , 2005 . 0471718130. Box, George E.P., Hunter, J Stuart and Hunter, William G. Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l. : John Wiley & Sons, Inc, 2005. 0471718130."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2365952.2365954"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557139"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1964897.1964905"},{"key":"e_1_3_2_1_24_1","volume-title":"Insanely Simple: The Obsession That Drives Apple's Success. s.l. : Portfolio Hardcover","author":"Segall Ken","year":"2012","unstructured":"Segall , Ken . Insanely Simple: The Obsession That Drives Apple's Success. s.l. : Portfolio Hardcover , 2012 . 978-1591844839. Segall, Ken. Insanely Simple: The Obsession That Drives Apple's Success. s.l. : Portfolio Hardcover, 2012. 978-1591844839."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2010.264"},{"key":"e_1_3_2_1_27_1","volume-title":"Experimentation and Testing: A Primer. Occam's Razor. {Online}","author":"Kaushik Avinash","year":"2006","unstructured":"Kaushik , Avinash . Experimentation and Testing: A Primer. Occam's Razor. {Online} May 22, 2006 . http:\/\/www.kaushik.net\/avinash\/2006\/05\/experimentation-and-testing-a-primer.html. Kaushik, Avinash. Experimentation and Testing: A Primer. Occam's Razor. {Online} May 22, 2006. http:\/\/www.kaushik.net\/avinash\/2006\/05\/experimentation-and-testing-a-primer.html."},{"key":"e_1_3_2_1_28_1","volume-title":"Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press","author":"Moran Mike","year":"2007","unstructured":"Moran , Mike . Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press , 2007 . 0132255960. Moran, Mike. Do It Wrong Quickly: How the Web Changes the Old Marketing Rules . s.l. : IBM Press, 2007. 0132255960."},{"key":"e_1_3_2_1_29_1","volume-title":"Testing to Cull the Living Flower. {Online}","author":"McKinley Dan","year":"2013","unstructured":"McKinley , Dan . Testing to Cull the Living Flower. {Online} Jan 2013 . http:\/\/mcfunley.com\/testing-to-cull-the-living-flower. McKinley, Dan. Testing to Cull the Living Flower. {Online} Jan 2013. http:\/\/mcfunley.com\/testing-to-cull-the-living-flower."},{"key":"e_1_3_2_1_30_1","volume-title":"S Stanley and Karr, Allan. 3","author":"Deming","year":"2011","unstructured":"Deming , data and observational studies: A process out of control and needing fixing. Young , S Stanley and Karr, Allan. 3 , 2011 , Significance , Vol . 8. http:\/\/www.niss.org\/sites\/default\/files\/Young%20Karr%20Obs%20Study%20Problem.pdf. Deming, data and observational studies: A process out of control and needing fixing. Young , S Stanley and Karr, Allan. 3, 2011, Significance, Vol. 8. http:\/\/www.niss.org\/sites\/default\/files\/Young%20Karr%20Obs%20Study%20Problem.pdf."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Why Most Published Research Findings Are False. Ioannidis John P. 8 2005 PLoS Medicine Vol. 2 p. e124. http:\/\/www.plosmedicine.org\/article\/info:doi\/10.1371\/journal.pmed.0020124.  Why Most Published Research Findings Are False. Ioannidis John P. 8 2005 PLoS Medicine Vol. 2 p. e124. http:\/\/www.plosmedicine.org\/article\/info:doi\/10.1371\/journal.pmed.0020124.","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_1_32_1","volume-title":"s.l. : The Journal of the American Medical Association","author":"Highly Cited Clinical Research Ioannidis","year":"2005","unstructured":"Contradicted and Initially Stronger Effects in Highly Cited Clinical Research . Ioannidis , John P . 2 , s.l. : The Journal of the American Medical Association , 2005 , Vol. 294 . http:\/\/jama.jamanetwork.com\/article.aspx?articleid=201218. Contradicted and Initially Stronger Effects in Highly Cited Clinical Research. Ioannidis, John P. 2, s.l. : The Journal of the American Medical Association, 2005, Vol. 294. http:\/\/jama.jamanetwork.com\/article.aspx?articleid=201218."},{"key":"e_1_3_2_1_33_1","volume-title":"Delinquency. Washington Post.","author":"Weiss Rick","year":"2007","unstructured":"Weiss , Rick . Study Debunks Theory On Teen Sex , Delinquency. Washington Post. 2007 , Nov 11. http:\/\/www.washingtonpost.com\/wp-dyn\/content\/story\/2007\/11\/11\/ST2007111100542.html. Weiss, Rick. Study Debunks Theory On Teen Sex, Delinquency. Washington Post. 2007, Nov 11. http:\/\/www.washingtonpost.com\/wp-dyn\/content\/story\/2007\/11\/11\/ST2007111100542.html."},{"key":"e_1_3_2_1_34_1","unstructured":"Myopia How Science Works: The Story of Night-Light. Myopia: Prevention and Control. {Online} http:\/\/www.myopiaprevention.org\/references_night_lights.html.  Myopia How Science Works: The Story of Night-Light. Myopia: Prevention and Control. {Online} http:\/\/www.myopiaprevention.org\/references_night_lights.html."},{"key":"e_1_3_2_1_35_1","volume-title":"Online Controlled Experiments: Listening to the Customers, not to the HiPPO. Keynote at EC10: the 11th ACM Conference on Electronic Commerce","author":"Kohavi Ron","year":"2010","unstructured":"Kohavi , Ron . Online Controlled Experiments: Listening to the Customers, not to the HiPPO. Keynote at EC10: the 11th ACM Conference on Electronic Commerce . 2010 . http:\/\/www.exp-platform.com\/Documents\/2010-06%20EC10.pptx. Kohavi, Ron. Online Controlled Experiments: Listening to the Customers, not to the HiPPO. Keynote at EC10: the 11th ACM Conference on Electronic Commerce. 2010. http:\/\/www.exp-platform.com\/Documents\/2010-06%20EC10.pptx."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.7208\/chicago\/9780226458106.001.0001"},{"key":"e_1_3_2_1_37_1","unstructured":"Wikipedia. Semmelweis reflex. http:\/\/en.wikipedia.org\/wiki\/Semmelweis_reflex.  Wikipedia. Semmelweis reflex. http:\/\/en.wikipedia.org\/wiki\/Semmelweis_reflex."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1002\/9781118983836"},{"key":"e_1_3_2_1_39_1","volume-title":"Bad Medicine: Doctors Doing Harm Since Hippocrates. s.l. : Oxford University Press","author":"Wooton David","year":"2007","unstructured":"Wooton , David . Bad Medicine: Doctors Doing Harm Since Hippocrates. s.l. : Oxford University Press , 2007 . Wooton, David. Bad Medicine: Doctors Doing Harm Since Hippocrates. s.l. : Oxford University Press, 2007."},{"key":"e_1_3_2_1_40_1","volume-title":"Make Data Useful. {Online}","author":"Linden Greg","year":"2006","unstructured":"Linden , Greg . Make Data Useful. {Online} Dec 2006 . home.blarg.net\/~glinden\/StanfordDataMining.2006-11-29.ppt. Linden, Greg. Make Data Useful. {Online} Dec 2006. home.blarg.net\/~glinden\/StanfordDataMining.2006-11-29.ppt."},{"key":"e_1_3_2_1_41_1","volume-title":"s.l. : Velocity 09: Velocity Web Performance and Operations Conference","author":"User Impact Schurman","year":"2009","unstructured":"Performance Related Changes and their User Impact . Schurman , Eric and Brutlag , Jake . s.l. : Velocity 09: Velocity Web Performance and Operations Conference , 2009 . Performance Related Changes and their User Impact. Schurman, Eric and Brutlag, Jake. s.l. : Velocity 09: Velocity Web Performance and Operations Conference, 2009."},{"key":"e_1_3_2_1_42_1","unstructured":"Douglas Bowman. Goodbye Google. StopDesign. {Online} http:\/\/stopdesign.com\/archive\/2009\/03\/20\/goodbye-google.html.  Douglas Bowman. Goodbye Google. StopDesign. {Online} http:\/\/stopdesign.com\/archive\/2009\/03\/20\/goodbye-google.html."},{"key":"e_1_3_2_1_43_1","first-page":"35","volume":"3","author":"Clinical Trials O'Brien","year":"1979","unstructured":"A Multiple Testing Procedure for Clinical Trials . O'Brien , Peter C . and Fleming , Thomas R. 3 , September 1979 , Biometrics, Vol. 35 , pp. 549--556. A Multiple Testing Procedure for Clinical Trials. O'Brien, Peter C. and Fleming, Thomas R. 3, September 1979, Biometrics, Vol. 35, pp. 549--556.","journal-title":"Thomas R."},{"key":"e_1_3_2_1_44_1","volume-title":"s.l. : Bmc Bioinformatics","author":"Strimmer Korbinian","year":"2008","unstructured":"A unified approach to false discovery rate estimation. Strimmer , Korbinian . 1 , s.l. : Bmc Bioinformatics , 2008 , Vol. 9 . A unified approach to false discovery rate estimation. Strimmer, Korbinian. 1, s.l. : Bmc Bioinformatics, 2008, Vol. 9."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433413"}],"event":{"name":"KDD' 13: The 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","location":"Chicago Illinois USA","acronym":"KDD' 13","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 19th ACM SIGKDD international conference on Knowledge discovery and data mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2487575.2488217","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2487575.2488217","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:39:29Z","timestamp":1750235969000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2487575.2488217"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,8,11]]},"references-count":45,"alternative-id":["10.1145\/2487575.2488217","10.1145\/2487575"],"URL":"https:\/\/doi.org\/10.1145\/2487575.2488217","relation":{},"subject":[],"published":{"date-parts":[[2013,8,11]]},"assertion":[{"value":"2013-08-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}