{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T03:11:07Z","timestamp":1759893067674,"version":"build-2065373602"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T00:00:00Z","timestamp":1746662400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,5,8]]},"DOI":"10.1145\/3701716.3715241","type":"proceedings-article","created":{"date-parts":[[2025,5,23]],"date-time":"2025-05-23T16:12:56Z","timestamp":1748016776000},"page":"374-383","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Know When to Fold: Futility-Aware Early Termination in Online Experiments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2450-1964","authenticated-orcid":false,"given":"Yu","family":"Liu","sequence":"first","affiliation":[{"name":"Amazon, Seattle, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7820-4271","authenticated-orcid":false,"given":"Runzhe","family":"Wan","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-7328-423X","authenticated-orcid":false,"given":"Yian","family":"Huang","sequence":"additional","affiliation":[{"name":"Columbia Univeristy, New York, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6636-8488","authenticated-orcid":false,"given":"James","family":"McQueen","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6413-1969","authenticated-orcid":false,"given":"Doug","family":"Hains","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-5439-1469","authenticated-orcid":false,"given":"Jinxiang","family":"Gu","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1875-2115","authenticated-orcid":false,"given":"Rui","family":"Song","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,5,23]]},"reference":[{"issue":"2","key":"e_1_3_2_2_1_1","first-page":"261","article-title":"Sample size determination: a review","volume":"46","author":"Adcock C. J.","year":"1997","unstructured":"C. J. Adcock. 1997. Sample size determination: a review. Journal of the Royal Statistical Society: Series D (The Statistician), 46(2), 261--283.","journal-title":"Journal of the Royal Statistical Society: Series D (The Statistician)"},{"key":"e_1_3_2_2_2_1","unstructured":"Jean-Yves Audibert S\u00e9bastien Bubeck and R\u00e9mi Munos. 2010. Best arm identification in multi-armed bandits. In COLT 41--53."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"Rebecca A. Betensky. 1997. Early stopping to accept H0 based on conditional power: approximations and comparisons. Biometrics 794--806.","DOI":"10.2307\/2533543"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939785"},{"volume-title":"Statistical power analysis for the behavioral sciences","author":"Cohen Jacob","key":"e_1_3_2_2_5_1","unstructured":"Jacob Cohen. 2013. Statistical power analysis for the behavioral sciences. Academic Press."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"crossref","unstructured":"David L. Demets and K. K. Gordon Lan. 1994. Interim analysis: the alpha spending function approach. Statistics in Medicine 13(13--14) 1341--1352.","DOI":"10.1002\/sim.4780131308"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2742563"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1002\/sim.2204"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1080\/00273171.2016.1191324"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/70.3.659"},{"volume-title":"The theory of probability","author":"Jeffreys Harold","key":"e_1_3_2_2_12_1","unstructured":"Harold Jeffreys. 1961. The theory of probability. Oxford University Press."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3097992"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.2135"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.4103\/0974-7788.59946"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1037\/1082-989X.11.4.363"},{"key":"e_1_3_2_2_17_1","first-page":"39","article-title":"Online experimentation at Microsoft","volume":"11","author":"Kohavi Ronny","year":"2009","unstructured":"Ronny Kohavi, Thomas Crook, Roger Longbotham, Brian Frasca, Randy Henne, Juan Lavista Ferres, and Tamir Melamed. 2009. Online experimentation at Microsoft. Data Mining Case Studies, 11, 39.","journal-title":"Data Mining Case Studies"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1080\/07474946.2011.539924"},{"key":"e_1_3_2_2_20_1","unstructured":"Madan Gopal Kundu Sandipan Samanta and Shoubhik Mondal. 2021. Conditional power predictive power and probability of success in clinical trials with continuous binary and time-to-event endpoints."},{"key":"e_1_3_2_2_21_1","unstructured":"Madan Gopal Kundu Sandipan Samanta and Shoubhik Mondal. 2023. Review of calculation of conditional power predictive power and probability of success in clinical trials with continuous binary and time-to-event endpoints."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1002\/sim.2151"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"crossref","unstructured":"Daniel Lakens Friedrich Pahlke and Gernot Wassmer. 2021. Group sequential designs: A tutorial.","DOI":"10.31234\/osf.io\/x4azm"},{"issue":"2","key":"e_1_3_2_2_24_1","first-page":"129","article-title":"The choice of sample size","volume":"46","author":"Lindley Dennis V.","year":"1997","unstructured":"Dennis V. Lindley. 1997. The choice of sample size. Journal of the Royal Statistical Society: Series D (The Statistician), 46(2), 129--138.","journal-title":"Journal of the Royal Statistical Society: Series D (The Statistician)"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i12.29313"},{"key":"e_1_3_2_2_26_1","volume-title":"The KDD'23 Workshop on Causal Discovery, Prediction and Decision, PMLR, 25--48","author":"Masoero Lorenzo","year":"2023","unstructured":"Lorenzo Masoero, Doug Hains, and James McQueen. 2023. Leveraging covariate adjustments at scale in online A\/B testing. In The KDD'23 Workshop on Causal Discovery, Prediction and Decision, PMLR, 25--48."},{"key":"e_1_3_2_2_27_1","volume-title":"International Conference on Artificial Intelligence and Statistics, PMLR, 1775--1785","author":"Richardson Thomas S.","year":"2022","unstructured":"Thomas S. Richardson, Yu Liu, James McQueen, and Doug Hains. 2022. A Bayesian model for online activity sample sizes. In International Conference on Artificial Intelligence and Statistics, PMLR, 1775--1785."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177696786"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1037\/met0000061"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1214\/22-AOAS1700"},{"volume-title":"Breakthroughs in Statistics: Foundations and Basic Theory","author":"Wald Abraham","key":"e_1_3_2_2_31_1","unstructured":"Abraham Wald. 1992. Sequential tests of statistical hypotheses. In Breakthroughs in Statistics: Foundations and Basic Theory, Springer, 256--298."},{"key":"e_1_3_2_2_32_1","unstructured":"Runzhe Wan Yu Liu James McQueen Doug Hains and Rui Song. 2023. Experimentation platforms meet reinforcement learning: Bayesian sequential decision-making for continuous monitoring. KDD."},{"issue":"2","key":"e_1_3_2_2_33_1","first-page":"185","article-title":"Bayesian sample size calculations for hypothesis testing","volume":"46","author":"Weiss Robert","year":"1997","unstructured":"Robert Weiss. 1997. Bayesian sample size calculations for hypothesis testing. Journal of the Royal Statistical Society: Series D (The Statistician), 46(2), 185--191.","journal-title":"Journal of the Royal Statistical Society: Series D (The Statistician)"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"crossref","unstructured":"Bernard L. Welch. 1947. The generalization of 'STUDENT'S' problem when several different population variances are involved. Biometrika 34(1--2) 28--35.","DOI":"10.1093\/biomet\/34.1-2.28"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939733"}],"event":{"name":"WWW '25: The ACM Web Conference 2025","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Sydney NSW Australia","acronym":"WWW '25"},"container-title":["Companion Proceedings of the ACM on Web Conference 2025"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3701716.3715241","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3701716.3715241","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T02:59:33Z","timestamp":1759892373000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3701716.3715241"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,8]]},"references-count":35,"alternative-id":["10.1145\/3701716.3715241","10.1145\/3701716"],"URL":"https:\/\/doi.org\/10.1145\/3701716.3715241","relation":{},"subject":[],"published":{"date-parts":[[2025,5,8]]},"assertion":[{"value":"2025-05-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}