{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T07:09:31Z","timestamp":1781334571742,"version":"3.54.1"},"reference-count":44,"publisher":"Informa UK Limited","issue":"4","content-domain":{"domain":["www.tandfonline.com"],"crossmark-restriction":true},"short-container-title":["Technometrics"],"published-print":{"date-parts":[[2022,10,2]]},"DOI":"10.1080\/00401706.2022.2124309","type":"journal-article","created":{"date-parts":[[2022,9,23]],"date-time":"2022-09-23T12:27:28Z","timestamp":1663936048000},"page":"524-534","update-policy":"https:\/\/doi.org\/10.1080\/tandf_crossmark_01","source":"Crossref","is-referenced-by-count":13,"title":["Novelty and Primacy: A Long-Term Estimator for Online Experiments"],"prefix":"10.1080","volume":"64","author":[{"given":"Soheil","family":"Sadeghi","sequence":"first","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Somit","family":"Gupta","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Stefan","family":"Gramatovici","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8839-6024","authenticated-orcid":false,"given":"Jiannan","family":"Lu","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hao","family":"Ai","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruhan","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"301","published-online":{"date-parts":[[2022,11,8]]},"reference":[{"key":"e_1_3_3_2_1","doi-asserted-by":"publisher","DOI":"10.1111\/0034-6527.00321"},{"key":"e_1_3_3_3_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0046719"},{"key":"e_1_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-0262.2006.00668.x"},{"key":"e_1_3_3_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0010-9452(08)70008-X"},{"key":"e_1_3_3_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joep.2018.05.002"},{"key":"e_1_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0054388"},{"key":"e_1_3_3_8_1","doi-asserted-by":"crossref","unstructured":"Chen N. Liu M. and Xu Y. (2019) \u201cHow a\/b Tests could Go Wrong: Automatic Diagnosis of Invalid Online Experiments \u201d in Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining pp. 501\u2013509.","DOI":"10.1145\/3289600.3291000"},{"key":"e_1_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1162\/REST_a_00049"},{"key":"e_1_3_3_10_1","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.1999.10474445"},{"key":"e_1_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.2014.16153"},{"key":"e_1_3_3_12_1","doi-asserted-by":"crossref","unstructured":"Dmitriev P. Frasca B. Gupta S. Kohavi R. and Vaz G. (2016) \u201cPitfalls of Long-Term Online Controlled Experiments \u201d in 2016 IEEE International Conference on Big Data pp. 1367\u20131376. IEEE.","DOI":"10.1109\/BigData.2016.7840744"},{"key":"e_1_3_3_13_1","doi-asserted-by":"crossref","unstructured":"Dmitriev P. Gupta S. Kim D. W. and Vaz G. (2017) \u201cA Dirty Dozen: Twelve Common Cetric Interpretation Pitfalls in Online Controlled Experiments \u201d in Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining pp. 1427\u20131436.","DOI":"10.1145\/3097983.3098024"},{"key":"e_1_3_3_14_1","doi-asserted-by":"publisher","DOI":"10.1162\/rest.89.2.221"},{"key":"e_1_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0058559"},{"key":"e_1_3_3_16_1","first-page":"174","article-title":"\u201cThe Novelty Effect of New Soccer Stadia: The Case of Germany","volume":"1","author":"Feddersen A.","year":"2006","unstructured":"Feddersen, A., Maennig, W., Borcherding, M. (2006), \u201cThe Novelty Effect of New Soccer Stadia: The Case of Germany,\u201d International Journal of Sport Finance, 1, 174\u2013188.","journal-title":"International Journal of Sport Finance"},{"key":"e_1_3_3_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3320093"},{"key":"e_1_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1080\/00401706.2019.1701556"},{"key":"e_1_3_3_19_1","doi-asserted-by":"crossref","unstructured":"Hohnhold H. O\u2019Brien D. and Tang D. (2015) \u201cFocusing on the Long-Term: It\u2019s Good for Users and Business \u201d in Proceedings of the 21th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining pp. 1849\u20131858.","DOI":"10.1145\/2783258.2788583"},{"key":"e_1_3_3_20_1","first-page":"111","article-title":"\u201cAn Empirical Review of the Stadium Novelty Effect","volume":"12","author":"Howard D. R.","year":"2003","unstructured":"Howard, D. R., and Crompton, J. L. (2003), \u201cAn Empirical Review of the Stadium Novelty Effect,\u201d Sport Marketing Quarterly, 12, 111\u2013116.","journal-title":"Sport Marketing Quarterly"},{"key":"e_1_3_3_21_1","volume-title":"Social, and Biomedical Sciences","author":"Imbens G. W.","year":"2015","unstructured":"Imbens, G. W., and Rubin, D. B. (2015), Causal Inference in Statistics, Social, and Biomedical Sciences, Cambridge: Cambridge University Press."},{"key":"e_1_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0026818"},{"key":"e_1_3_3_23_1","unstructured":"Kaufman R. L. Pitchforth J. and Vermeer L. (2017) \u201cDemocratizing Online Controlled Experiments at booking.com \u201d arXiv preprint arXiv:1710.08217."},{"key":"e_1_3_3_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339653"},{"key":"e_1_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7687-1_891"},{"key":"e_1_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_3_27_1","unstructured":"Kohavi R. and Round M. (2004) \u201cEmetrics Summit 2004: Front Line Internet Analytics at amazon.com \u201d Amazon.com Copyright."},{"key":"e_1_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1017\/9781108653985"},{"key":"e_1_3_3_29_1","doi-asserted-by":"publisher","DOI":"10.1002\/cb.291"},{"key":"e_1_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00011"},{"key":"e_1_3_3_31_1","doi-asserted-by":"crossref","unstructured":"Mutsuddi A. U. and Connelly K. (2012) \u201cText Messages for Encouraging Physical Activity: Are they Effective after the Novelty Effect Wears Off?\u201d in 2012 6th International Conference on Pervasive Computing Technologies for Healthcare (PervasiveHealth) and Workshops pp. 33\u201340. IEEE.","DOI":"10.4108\/icst.pervasivehealth.2012.248715"},{"key":"e_1_3_3_32_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0024139"},{"key":"e_1_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0019900"},{"key":"e_1_3_3_34_1","unstructured":"Roth J. and Sant\u2019Anna P. H. (2021) \u201cEfficient Estimation for Staggered Rollout Designs \u201d arXiv preprint arXiv:2102.01291."},{"key":"e_1_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.1080\/00401706.2019.1647288"},{"key":"e_1_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.1002\/asmb.874"},{"key":"e_1_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1002\/asmb.2104"},{"key":"e_1_3_3_38_1","unstructured":"Shen M. and Lee M. (2017) \u201cSelection Bias in Online Experimentation \u201d Available at https:\/\/medium.com\/airbnb-engineering\/selection-bias-in-online-experimentation-c3d67795cceb."},{"key":"e_1_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.26.6.1589"},{"key":"e_1_3_3_40_1","doi-asserted-by":"crossref","unstructured":"Tang D. Agarwal A. O\u2019Brien D. and Meyer M. (2010) \u201cOverlapping Experiment Infrastructure: More Better Faster Experimentation \u201d in Proceedings of the 16th ACM SIGKDD International Conference on Knowledge Discovery and D ata Mining pp. 17\u201326.","DOI":"10.1145\/1835804.1835810"},{"key":"e_1_3_3_41_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0092987"},{"key":"e_1_3_3_42_1","unstructured":"Urban S. Sreenivasan R. and Kannan V. (2016) \u201cIt is all A\/Bout Testing: The Netflix Experimentation Platform \u201d https:\/\/medium.com\/netflix-techblog\/its-all-a-bout-testing-the-netflixexperimentation-platform-4e1ca458c15."},{"key":"e_1_3_3_43_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.electstud.2016.09.002"},{"key":"e_1_3_3_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357384.3358143"},{"key":"e_1_3_3_45_1","volume-title":"Experiments: Planning, Analysis, and Optimization","author":"Wu C. J.","year":"2011","unstructured":"Wu, C. J., and Hamada, M. S. (2011), Experiments: Planning, Analysis, and Optimization (Vol. 552), New York: Wiley."}],"container-title":["Technometrics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.tandfonline.com\/doi\/pdf\/10.1080\/00401706.2022.2124309","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,28]],"date-time":"2023-11-28T18:56:00Z","timestamp":1701197760000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.tandfonline.com\/doi\/full\/10.1080\/00401706.2022.2124309"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,10,2]]},"references-count":44,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2022,10,2]]}},"alternative-id":["10.1080\/00401706.2022.2124309"],"URL":"https:\/\/doi.org\/10.1080\/00401706.2022.2124309","relation":{},"ISSN":["0040-1706","1537-2723"],"issn-type":[{"value":"0040-1706","type":"print"},{"value":"1537-2723","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,10,2]]},"assertion":[{"value":"The publishing and review policy for this title is described in its Aims & Scope.","order":1,"name":"peerreview_statement","label":"Peer Review Statement"},{"value":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=utch20","URL":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=utch20","order":2,"name":"aims_and_scope_url","label":"Aim & Scope"},{"value":"2021-01-15","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2022-08-20","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2022-11-08","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}