{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,20]],"date-time":"2026-03-20T15:35:06Z","timestamp":1774020906135,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":32,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,30]],"date-time":"2023-04-30T00:00:00Z","timestamp":1682812800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,30]]},"DOI":"10.1145\/3543873.3584635","type":"proceedings-article","created":{"date-parts":[[2023,4,28]],"date-time":"2023-04-28T11:36:14Z","timestamp":1682681774000},"page":"396-400","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Anytime-Valid Confidence Sequences in an Enterprise A\/B Testing Platform"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8625-1844","authenticated-orcid":false,"given":"Akash","family":"Maharaj","sequence":"first","affiliation":[{"name":"Adobe, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0344-1368","authenticated-orcid":false,"given":"Ritwik","family":"Sinha","sequence":"additional","affiliation":[{"name":"Adobe Research, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9932-7657","authenticated-orcid":false,"given":"David","family":"Arbour","sequence":"additional","affiliation":[{"name":"Adobe Research, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4471-1405","authenticated-orcid":false,"given":"Ian","family":"Waudby-Smith","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6002-742X","authenticated-orcid":false,"given":"Simon Z.","family":"Liu","sequence":"additional","affiliation":[{"name":"Adobe, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5634-4345","authenticated-orcid":false,"given":"Moumita","family":"Sinha","sequence":"additional","affiliation":[{"name":"Adobe, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1478-083X","authenticated-orcid":false,"given":"Raghavendra","family":"Addanki","sequence":"additional","affiliation":[{"name":"Adobe Research, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0497-311X","authenticated-orcid":false,"given":"Aaditya","family":"Ramdas","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1060-8939","authenticated-orcid":false,"given":"Manas","family":"Garg","sequence":"additional","affiliation":[{"name":"Adobe, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1357-1502","authenticated-orcid":false,"given":"Viswanathan","family":"Swaminathan","sequence":"additional","affiliation":[{"name":"Adobe Research, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,4,30]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Adobe. 2022. Experimentation panel Adobe Customer Journey Analytics. https:\/\/experienceleague.adobe.com\/docs\/analytics-platform\/using\/cja-workspace\/panels\/experimentation.html?lang=en. (Accessed on 11\/07\/2022)."},{"key":"e_1_3_2_1_2_1","unstructured":"Adobe. 2022. Get started with content experiment | Adobe Journey Optimizer. https:\/\/experienceleague.adobe.com\/docs\/journey-optimizer\/using\/campaigns\/content-experiment\/get-started-experiment.html. (Accessed on 11\/07\/2022)."},{"key":"e_1_3_2_1_3_1","unstructured":"Adobe. 2022. Statistical Calculations used by Adobe Journey Optimizer Experimentation | Adobe Journey Optimizer. https:\/\/experienceleague.adobe.com\/docs\/journey-optimizer\/using\/campaigns\/content-experiment\/experiment-calculations.html?lang=en. (Accessed on 11\/07\/2022)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.58.1.66"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.3758\/s13423-020-01803-x"},{"key":"e_1_3_2_1_6_1","volume-title":"Interim analysis: the alpha spending function approach. Statistics in medicine 13, 13-14","author":"Demets L","year":"1994","unstructured":"David\u00a0L Demets and KK\u00a0Gordon Lan. 1994. Interim analysis: the alpha spending function approach. Statistics in medicine 13, 13-14 (1994), 1341\u20131352."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_1_8_1","volume-title":"Adaptive Design Clinical Trials for Drugs and Biologics Guidance for Industry. Guidance Document FDA-2018-D-3124","author":"Food and Drug Administration","unstructured":"Food and Drug Administration. 2018. Adaptive Design Clinical Trials for Drugs and Biologics Guidance for Industry. Guidance Document FDA-2018-D-3124. U.S. Department of Health and Human Services."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/70.3.659"},{"key":"e_1_3_2_1_10_1","unstructured":"Technical Help. 2022. General methodology - optimize resource hub. https:\/\/support.google.com\/optimize\/answer\/7405543?hl=en#zippy=%2Cin-this-article"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1214\/20-AOS1991"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2743061"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3097992"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.2135"},{"key":"e_1_3_2_1_15_1","volume-title":"Conference on Learning Theory. PMLR","author":"Jun Kwang-Sung","year":"2019","unstructured":"Kwang-Sung Jun and Francesco Orabona. 2019. Parameter-free online convex optimization with sub-exponential noise. In Conference on Learning Theory. PMLR, 1802\u20131823."},{"key":"e_1_3_2_1_16_1","volume-title":"Trustworthy online controlled experiments: A practical guide to a\/b testing","author":"Kohavi Ron","unstructured":"Ron Kohavi, Diane Tang, and Ya Xu. 2020. Trustworthy online controlled experiments: A practical guide to a\/b testing. Cambridge University Press."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1214\/aop\/1176996135"},{"key":"e_1_3_2_1_18_1","volume-title":"On confidence sequences. The Annals of Statistics","author":"Lai Tze\u00a0Leung","year":"1976","unstructured":"Tze\u00a0Leung Lai. 1976. On confidence sequences. The Annals of Statistics (1976), 265\u2013280."},{"key":"e_1_3_2_1_19_1","volume-title":"Anytime-Valid Confidence Sequences in an Enterprise A\/B Testing Platform. arXiv preprint arXiv:2302.10108","author":"Maharaj V.","year":"2023","unstructured":"Akash\u00a0V. Maharaj, Ritwik Sinha, David Arbour, Ian Waudby-Smith, Simon\u00a0Z. Liu, Moumita Sinha, Raghavendra Addanki, Aaditya Ramdas, Manas Garg, and Viswanathan Swaminathan. 2023. Anytime-Valid Confidence Sequences in an Enterprise A\/B Testing Platform. arXiv preprint arXiv:2302.10108 (2023)."},{"key":"e_1_3_2_1_20_1","volume-title":"A multiple testing procedure for clinical trials. Biometrics","author":"O\u2019Brien C","year":"1979","unstructured":"Peter\u00a0C O\u2019Brien and Thomas\u00a0R Fleming. 1979. A multiple testing procedure for clinical trials. Biometrics (1979), 549\u2013556."},{"key":"e_1_3_2_1_21_1","volume-title":"Tight concentrations and confidence sequences from the regret of universal portfolio. arXiv preprint arXiv:2110.14099","author":"Orabona Francesco","year":"2021","unstructured":"Francesco Orabona and Kwang-Sung Jun. 2021. Tight concentrations and confidence sequences from the regret of universal portfolio. arXiv preprint arXiv:2110.14099 (2021)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/64.2.191"},{"key":"e_1_3_2_1_23_1","volume-title":"Game-theoretic statistics and safe anytime-valid inference. arXiv preprint arXiv:2210.01948","author":"Ramdas Aaditya","year":"2022","unstructured":"Aaditya Ramdas, Peter Gr\u00fcnwald, Vladimir Vovk, and Glenn Shafer. 2022. Game-theoretic statistics and safe anytime-valid inference. arXiv preprint arXiv:2210.01948 (2022)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177696786"},{"key":"e_1_3_2_1_25_1","volume-title":"Optional stopping: No problem for Bayesians. Psychonomic bulletin & review 21, 2","author":"Rouder N","year":"2014","unstructured":"Jeffrey\u00a0N Rouder. 2014. Optional stopping: No problem for Bayesians. Psychonomic bulletin & review 21, 2 (2014), 301\u2013308."},{"key":"e_1_3_2_1_26_1","volume-title":"Visual Website Optimizer","author":"Stucchio Chris","year":"2015","unstructured":"Chris Stucchio. 2015. Bayesian A\/B testing at VWO. Whitepaper, Visual Website Optimizer (2015)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177731118"},{"key":"e_1_3_2_1_28_1","volume-title":"Catoni-style confidence sequences for heavy-tailed mean estimation. arXiv preprint arXiv:2202.01250","author":"Wang Hongjian","year":"2022","unstructured":"Hongjian Wang and Aaditya Ramdas. 2022. Catoni-style confidence sequences for heavy-tailed mean estimation. arXiv preprint arXiv:2202.01250 (2022)."},{"key":"e_1_3_2_1_29_1","volume-title":"All of statistics: a concise course in statistical inference. Vol.\u00a026","author":"Wasserman Larry","unstructured":"Larry Wasserman. 2004. All of statistics: a concise course in statistical inference. Vol.\u00a026. Springer."},{"key":"e_1_3_2_1_30_1","volume-title":"Time-uniform central limit theory, asymptotic confidence sequences, and anytime-valid causal inference. arXiv preprint arXiv:2103.06476","author":"Waudby-Smith Ian","year":"2021","unstructured":"Ian Waudby-Smith, David Arbour, Ritwik Sinha, Edward\u00a0H Kennedy, and Aaditya Ramdas. 2021. Time-uniform central limit theory, asymptotic confidence sequences, and anytime-valid causal inference. arXiv preprint arXiv:2103.06476 (2021)."},{"key":"e_1_3_2_1_31_1","first-page":"20204","article-title":"Confidence sequences for sampling without replacement","volume":"33","author":"Waudby-Smith Ian","year":"2020","unstructured":"Ian Waudby-Smith and Aaditya Ramdas. 2020. Confidence sequences for sampling without replacement. Advances in Neural Information Processing Systems 33 (2020), 20204\u201320214.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_32_1","series-title":"Series B (to appear with discussion)","volume-title":"Estimating means of bounded random variables by betting. Journal of the Royal Statistical Society","author":"Waudby-Smith Ian","year":"2023","unstructured":"Ian Waudby-Smith and Aaditya Ramdas. 2023. Estimating means of bounded random variables by betting. Journal of the Royal Statistical Society, Series B (to appear with discussion) (2023)."}],"event":{"name":"WWW '23: The ACM Web Conference 2023","location":"Austin TX USA","acronym":"WWW '23","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Companion Proceedings of the ACM Web Conference 2023"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543873.3584635","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3543873.3584635","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T23:31:21Z","timestamp":1755819081000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543873.3584635"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,30]]},"references-count":32,"alternative-id":["10.1145\/3543873.3584635","10.1145\/3543873"],"URL":"https:\/\/doi.org\/10.1145\/3543873.3584635","relation":{},"subject":[],"published":{"date-parts":[[2023,4,30]]},"assertion":[{"value":"2023-04-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}