{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T19:32:25Z","timestamp":1768591945312,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","license":[{"start":{"date-parts":[[2009,6,28]],"date-time":"2009-06-28T00:00:00Z","timestamp":1246147200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2009,6,28]]},"DOI":"10.1145\/1557019.1557139","type":"proceedings-article","created":{"date-parts":[[2009,6,30]],"date-time":"2009-06-30T13:10:54Z","timestamp":1246367454000},"page":"1105-1114","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":89,"title":["Seven pitfalls to avoid when running controlled experiments on the web"],"prefix":"10.1145","author":[{"given":"Thomas","family":"Crook","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Brian","family":"Frasca","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ron","family":"Kohavi","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Roger","family":"Longbotham","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2009,6,28]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Enemy: Tracking Botnets. The Honeynet Project. {Online}","author":"Bacher Paul","year":"2005","unstructured":"Bacher , Paul , et al. 2005 . Know your Enemy: Tracking Botnets. The Honeynet Project. {Online} March 13, 2005. http:\/\/www.honeynet.org\/papers\/bots\/. Bacher, Paul, et al. 2005. Know your Enemy: Tracking Botnets. The Honeynet Project. {Online} March 13, 2005. http:\/\/www.honeynet.org\/papers\/bots\/."},{"key":"e_1_3_2_2_2_1","volume-title":"Web Robot Detection - Preprocessing Web Logfiles for Robot Detection. {book auth.} Maurizio Vichi, et al. New Developments in Classification and Data Analysis. s.l. : Springer","author":"Bomhardt Christian","year":"2005","unstructured":"Bomhardt , Christian , Gaul , Wolfgang and Schmidt-Thieme , Lars . 2005. Web Robot Detection - Preprocessing Web Logfiles for Robot Detection. {book auth.} Maurizio Vichi, et al. New Developments in Classification and Data Analysis. s.l. : Springer , 2005 . Bomhardt, Christian, Gaul, Wolfgang and Schmidt-Thieme, Lars. 2005. Web Robot Detection - Preprocessing Web Logfiles for Robot Detection. {book auth.} Maurizio Vichi, et al. New Developments in Classification and Data Analysis. s.l. : Springer, 2005."},{"key":"e_1_3_2_2_3_1","first-page":"0471718130","article-title":"Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l. : John Wiley&Sons","volume":"2005","author":"Box George E.P.","year":"2005","unstructured":"Box , George E.P. , Hunter , J Stuart and Hunter , William G . 2005 . Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l. : John Wiley&Sons , Inc , 2005. 0471718130 . Box, George E.P., Hunter, J Stuart and Hunter, William G. 2005. Statistics for Experimenters: Design, Innovation, and Discovery. 2nd. s.l. : John Wiley&Sons, Inc, 2005. 0471718130.","journal-title":"Inc"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/4236.968829"},{"key":"e_1_3_2_2_5_1","volume-title":"Tibshirani","author":"Efron Bradley","year":"1993","unstructured":"Efron , Bradley and Robert J . Tibshirani . 1993 . An Introduction to the Bootstrap. New York : Chapman &Hall, 1993. 0-412-04231-2. Efron, Bradley and Robert J. Tibshirani. 1993. An Introduction to the Bootstrap. New York : Chapman&Hall, 1993. 0-412-04231-2."},{"key":"e_1_3_2_2_6_1","first-page":"1","volume-title":"The Biological Standardization of Insulin. Supplement to the Journal of the Royal Statistical Society","author":"Fieller E C","year":"1940","unstructured":"Fieller , E C . 1940. The Biological Standardization of Insulin. Supplement to the Journal of the Royal Statistical Society . 1940 , Vol. 7 , 1, pp. 1 -- 64 . Fieller, E C. 1940. The Biological Standardization of Insulin. Supplement to the Journal of the Royal Statistical Society. 1940, Vol. 7, 1, pp. 1--64."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1059981.1059982"},{"key":"e_1_3_2_2_8_1","volume-title":"Customer Satisfaction: The Customer Experience Through the Customer's Eyes. s.l.","author":"Hill Nigel","year":"2007","unstructured":"Hill , Nigel , Roche , Greg and Allen , Rachel . 2007 . Customer Satisfaction: The Customer Experience Through the Customer's Eyes. s.l. : Cogent Publishing , 2007. Hill, Nigel, Roche, Greg and Allen, Rachel. 2007. Customer Satisfaction: The Customer Experience Through the Customer's Eyes. s.l. : Cogent Publishing, 2007."},{"key":"e_1_3_2_2_9_1","volume-title":"Scientific Advertising. New York City : Crown Publishers Inc","author":"Hopkins Claude","year":"1923","unstructured":"Hopkins , Claude . 1923. Scientific Advertising. New York City : Crown Publishers Inc ., 1923 . Hopkins, Claude. 1923. Scientific Advertising. New York City : Crown Publishers Inc., 1923."},{"key":"e_1_3_2_2_10_1","volume-title":"2nd. s.l.","author":"Keppel Geoffrey","year":"1992","unstructured":"Keppel , Geoffrey , Saufley , William H and Tokunaga , Howard . 1992. Introduction to Design and Analysis . 2nd. s.l. : W.H. Freeman and Company , 1992 . Keppel, Geoffrey, Saufley, William H and Tokunaga, Howard. 1992. Introduction to Design and Analysis. 2nd. s.l. : W.H. Freeman and Company, 1992."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1023\/B:MACH.0000035473.11134.83"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281295"},{"key":"e_1_3_2_2_14_1","first-page":"114","article-title":"The New Mantra","volume":"11","author":"Koselka Rita","year":"1996","unstructured":"Koselka , Rita . 1996 . The New Mantra : MVT. Forbes . March 11 , 1996, pp. 114 -- 118 . Koselka, Rita. 1996. The New Mantra: MVT. Forbes. March 11, 1996, pp. 114--118.","journal-title":"MVT. Forbes"},{"key":"e_1_3_2_2_15_1","volume-title":"Stanford Encyclopedia of Philosophy. {Online}","author":"Malinas Gary","year":"2004","unstructured":"Malinas , Gary and Bigelow , John . 2004. Simpson's Paradox . Stanford Encyclopedia of Philosophy. {Online} 2004 . {Cited: February 28, 2008.} http:\/\/plato.stanford.edu\/entries\/paradox-simpson\/. Malinas, Gary and Bigelow, John. 2004. Simpson's Paradox. Stanford Encyclopedia of Philosophy. {Online} 2004. {Cited: February 28, 2008.} http:\/\/plato.stanford.edu\/entries\/paradox-simpson\/."},{"key":"e_1_3_2_2_16_1","volume-title":"Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley&Sons","author":"Mason Robert L","year":"1989","unstructured":"Mason , Robert L , Gunst , Richard F and Hess , James L . 1989. Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley&Sons , 1989 . 047185364X . Mason, Robert L, Gunst, Richard F and Hess, James L. 1989. Statistical Design and Analysis of Experiments With Applications to Engineering and Science. s.l. : John Wiley&Sons, 1989. 047185364X ."},{"key":"e_1_3_2_2_17_1","volume-title":"Design and Analysis of Experiments","author":"Montgomery Douglas C","year":"2005","unstructured":"Montgomery , Douglas C . 2005. Design and Analysis of Experiments . 6 th edition. s.l. : John Wiley &Sons, Inc, 2005 . 0-471-66159-7. Montgomery, Douglas C. 2005. Design and Analysis of Experiments. 6th edition. s.l. : John Wiley&Sons, Inc, 2005. 0-471-66159-7.","edition":"6"},{"key":"e_1_3_2_2_18_1","volume-title":"2nd. s.l. : John Wiley&Sons","author":"Rao C. Radhakrishna.","year":"1973","unstructured":"Rao , C. Radhakrishna. 1973. Linear Statistical Inference and Its Applications . 2nd. s.l. : John Wiley&Sons , Inc ., 1973 . Rao, C. Radhakrishna. 1973. Linear Statistical Inference and Its Applications. 2nd. s.l. : John Wiley&Sons, Inc., 1973."},{"key":"e_1_3_2_2_19_1","first-page":"0","article-title":"Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley&Sons","volume":"2001","author":"Roy Ranjit K","year":"2001","unstructured":"Roy , Ranjit K . 2001 . Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley&Sons , Inc , 2001. 0 - 471 -36101-1. Roy, Ranjit K. 2001. Design of Experiments using the Taguchi Approach : 16 Steps to Product and Process Improvement. s.l. : John Wiley&Sons, Inc, 2001. 0-471-36101-1.","journal-title":"Inc"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"crossref","first-page":"238","DOI":"10.1111\/j.2517-6161.1951.tb00088.x","article-title":"The Interpretation of Interaction in Contingency Tables","volume":"1951","author":"Simpson Edward H","year":"1951","unstructured":"Simpson , Edward H . 1951 . The Interpretation of Interaction in Contingency Tables . Journal of the Royal Statistical Society, Ser. B. 1951 , Vol. 13, pp. 238 -- 241 . Simpson, Edward H. 1951. The Interpretation of Interaction in Contingency Tables. Journal of the Royal Statistical Society, Ser. B. 1951, Vol. 13, pp. 238--241.","journal-title":"Journal of the Royal Statistical Society, Ser. B."},{"key":"e_1_3_2_2_21_1","first-page":"78","article-title":"Learning to Lead at Toyota","volume":"2004","author":"Spears Steven J","year":"2004","unstructured":"Spears , Steven J . 2004 . Learning to Lead at Toyota . Harvard Business Review . May 2004 , pp. 78 -- 86 . Spears, Steven J. 2004. Learning to Lead at Toyota. Harvard Business Review. May 2004, pp. 78--86.","journal-title":"Harvard Business Review"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1013228602957"},{"key":"e_1_3_2_2_23_1","volume-title":"Botnet","author":"Wikipedia","year":"2008","unstructured":"Wikipedia : Botnet . 2008 . Botnet. Wikipedia . {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Botnet. Wikipedia: Botnet. 2008. Botnet. Wikipedia. {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Botnet."},{"key":"e_1_3_2_2_24_1","volume-title":"Internet bot","author":"Wikipedia","year":"2008","unstructured":"Wikipedia : Internet bot . 2008 . Internet Bot. Wikipedia . {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Internet_bot. Wikipedia: Internet bot. 2008. Internet Bot. Wikipedia. {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Internet_bot."},{"key":"e_1_3_2_2_25_1","volume-title":"Simpson's Paradox","author":"Wikipedia","year":"2008","unstructured":"Wikipedia : Simpson's Paradox . 2008 . Simpson's paradox. Wikipedia . {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Simpson%27s_paradox. Wikipedia: Simpson's Paradox. 2008. Simpson's paradox. Wikipedia. {Online} 2008. {Cited: February 28, 2008.} http:\/\/en.wikipedia.org\/wiki\/Simpson%27s_paradox."}],"event":{"name":"KDD09: The 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","location":"Paris France","acronym":"KDD09","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","ACM Association for Computing Machinery"]},"container-title":["Proceedings of the 15th ACM SIGKDD international conference on Knowledge discovery and data mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1557019.1557139","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1557019.1557139","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T13:38:42Z","timestamp":1750253922000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1557019.1557139"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,6,28]]},"references-count":25,"alternative-id":["10.1145\/1557019.1557139","10.1145\/1557019"],"URL":"https:\/\/doi.org\/10.1145\/1557019.1557139","relation":{},"subject":[],"published":{"date-parts":[[2009,6,28]]},"assertion":[{"value":"2009-06-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}