{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,2]],"date-time":"2025-11-02T15:11:45Z","timestamp":1762096305969,"version":"build-2065373602"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,12]],"date-time":"2023-11-12T00:00:00Z","timestamp":1699747200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,12]]},"DOI":"10.1145\/3624062.3624202","type":"proceedings-article","created":{"date-parts":[[2023,11,10]],"date-time":"2023-11-10T13:53:39Z","timestamp":1699624419000},"page":"1288-1297","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Adaptive Stopping Rule for Performance Measurements"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6238-6709","authenticated-orcid":false,"given":"Viyom","family":"Mittal","sequence":"first","affiliation":[{"name":"Hewlett Packard Enterprise (HPE), USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6017-0816","authenticated-orcid":false,"given":"Pedro","family":"Bruel","sequence":"additional","affiliation":[{"name":"Hewlett Packard Enterprise (HPE), United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9830-8588","authenticated-orcid":false,"given":"Dejan","family":"Milojicic","sequence":"additional","affiliation":[{"name":"Hewlett Packard Enterprise (HPE), United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3709-1829","authenticated-orcid":false,"given":"Eitan","family":"Frachtenberg","sequence":"additional","affiliation":[{"name":"Hewlett Packard Enterprise (HPE), United States of America"}]}],"member":"320","published-online":{"date-parts":[[2023,11,12]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.32614\/RJ-2011-016"},{"key":"e_1_3_2_2_2_1","volume-title":"Statistics and Optimal Design for Benchmarking Experiments Involving User Traffic. In 24th International World Wide Web Conference (WWW\u201915)","author":"Bakshy Eytan","year":"2015","unstructured":"Eytan Bakshy and Eitan Frachtenberg. 2015. Statistics and Optimal Design for Benchmarking Experiments Involving User Traffic. In 24th International World Wide Web Conference (WWW\u201915). Florence, Italy, 108\u2013118."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSP49548.2020.9163456"},{"volume-title":"Pattern recognition and machine learning. Vol.\u00a04","author":"Bishop M","key":"e_1_3_2_2_4_1","unstructured":"Christopher\u00a0M Bishop and Nasser\u00a0M Nasrabadi. 2006. Pattern recognition and machine learning. Vol.\u00a04. Springer."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/IISWC.2009.5306797"},{"key":"e_1_3_2_2_6_1","volume-title":"On importance sampling in digital communications","author":"Chen J-C","year":"1993","unstructured":"J-C Chen, Dingqing Lu, John\u00a0S. Sadowsky, and Kung Yao. 1993. On importance sampling in digital communications.IEEE Journal on Selected Areas in Communications 11, 3 (4 1993), 289\u2013299."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.3758\/s13423-020-01803-x"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2016.33"},{"key":"e_1_3_2_2_9_1","volume-title":"Instruments, & Computers 30 (12","author":"Frick W","year":"1998","unstructured":"Robert\u00a0W Frick. 1998. A better stopping rule for conventional statistical tests. Behavior Research Methods, Instruments, & Computers 30 (12 1998), 690\u2013697."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1998561.1998569"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678687"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807644"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/WCNC.2013.6554882"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2016.2539167"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(79)90007-4"},{"key":"e_1_3_2_2_16_1","volume-title":"Doing Bayesian data analysis: A tutorial with R. JAGS, and Stan 2","author":"Kruschke John","year":"2014","unstructured":"John Kruschke. 2014. Doing Bayesian data analysis: A tutorial with R. JAGS, and Stan 2 (2014)."},{"key":"e_1_3_2_2_17_1","volume-title":"13th USENIX Symposium on Operating Systems Design and Implementation (OSDI). 409\u2013425","author":"Maricq Aleksander","year":"2018","unstructured":"Aleksander Maricq, Dmitry Duplyakin, Ivo Jimenez, Carlos Maltzahn, Ryan Stutsman, and Robert Ricci. 2018. Taming performance variability. In 13th USENIX Symposium on Operating Systems Design and Implementation (OSDI). 409\u2013425. www.usenix.org\/conference\/osdi18\/presentation\/maricq"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1988.10478581"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.3414\/ME11-02-0030"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TCOMM.2005.863780"},{"key":"e_1_3_2_2_21_1","volume-title":"Null hypothesis significance testing: a review of an old and continuing controversy. Psychological methods 5, 2","author":"Nickerson S","year":"2000","unstructured":"Raymond\u00a0S Nickerson. 2000. Null hypothesis significance testing: a review of an old and continuing controversy. Psychological methods 5, 2 (2000), 241."},{"key":"e_1_3_2_2_22_1","article-title":"Methodological principles for reproducible performance evaluation in cloud computing","volume":"47","author":"Papadopoulos V","year":"2019","unstructured":"Alessandro\u00a0V Papadopoulos, Laurens Versluis, Andr\u00e9 Bauer, Nikolas Herbst, J\u00f3akim Von\u00a0Kistowski, Ahmed Ali-Eldin, Cristina\u00a0L Abad, Jos\u00e9\u00a0Nelson Amaral, Petr Tuma, and Alexandru Iosup. 2019. Methodological principles for reproducible performance evaluation in cloud computing. Transactions on Software Engineering 47, 8 (8 2019), 1528\u20131543.","journal-title":"Transactions on Software Engineering"},{"key":"e_1_3_2_2_23_1","volume-title":"Optional stopping: No problem for Bayesians. Psychonomic bulletin & review 21 (4","author":"Rouder N","year":"2014","unstructured":"Jeffrey\u00a0N Rouder. 2014. Optional stopping: No problem for Bayesians. Psychonomic bulletin & review 21 (4 2014), 301\u2013308."},{"volume-title":"Extreme Value Theory and Machine Learning. Ph.\u00a0D. Dissertation","author":"Sabourin Anne","key":"e_1_3_2_2_24_1","unstructured":"Anne Sabourin. 2021. Extreme Value Theory and Machine Learning. Ph.\u00a0D. Dissertation. Institut polytechnique de Paris."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","unstructured":"Adam\u00a0N Sanborn and Thomas\u00a0T Hills. 2014. The frequentist implications of optional stopping on Bayesian hypothesis tests. 283\u2013300\u00a0pages. https:\/\/doi.org\/10.3758\/s13423-013-0518-9","DOI":"10.3758\/s13423-013-0518-9"},{"key":"e_1_3_2_2_26_1","volume-title":"Sequential hypothesis testing with Bayes factors: Efficiently testing mean differences.Psychological methods 22, 2","author":"Sch\u00f6nbrodt D","year":"2017","unstructured":"Felix\u00a0D Sch\u00f6nbrodt, Eric-Jan Wagenmakers, Michael Zehetleitner, and Marco Perugini. 2017. Sequential hypothesis testing with Bayes factors: Efficiently testing mean differences.Psychological methods 22, 2 (2017), 322."},{"key":"e_1_3_2_2_27_1","unstructured":"Mohammad Shahrad Rodrigo Fonseca Inigo Goiri Gohar Chaudhry Paul Batum Jason Cooke Eduardo Laureano Colby Tresness Mark Russinovich and Ricardo Bianchini. 2020. Serverless in the wild: Characterizing and optimizing the serverless workload at a large cloud provider. In 2020 USENIX annual technical conference (USENIX ATC 20). 205\u2013218."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2668930.2688052"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1287\/ijoc.1110.0471"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/IISWC.2005.1526010"},{"key":"e_1_3_2_2_31_1","volume-title":"Comparing computer adaptive testing stopping rules under the generalized partial-credit model. Behavior research methods 51 (6","author":"Stafford E","year":"2019","unstructured":"Rose\u00a0E Stafford, Christopher\u00a0R Runyon, Jodi\u00a0M Casabianca, and Barbara\u00a0G Dodd. 2019. Comparing computer adaptive testing stopping rules under the generalized partial-credit model. Behavior research methods 51 (6 2019), 1305\u20131320."},{"key":"e_1_3_2_2_32_1","volume-title":"17th symposium on networked systems design and implementation (NSDI). USENIX, 513\u2013527","author":"Uta Alexandru","year":"2020","unstructured":"Alexandru Uta, Alexandru Custura, Dmitry Duplyakin, Ivo Jimenez, Jan Rellermeyer, Carlos Maltzahn, Robert Ricci, and Alexandru Iosup. 2020. Is big data performance reproducible in modern cloud networks?. In 17th symposium on networked systems design and implementation (NSDI). USENIX, 513\u2013527. www.usenix.org\/conference\/nsdi20\/presentation\/uta"},{"key":"e_1_3_2_2_33_1","volume-title":"An experimental evaluation of a stopping rule aimed at maximizing cost-quality trade-offs in surveys. Journal of the Royal Statistical Society Series A: Statistics in Society","author":"Wagner James","year":"2023","unstructured":"James Wagner, Xinyu Zhang, Michael\u00a0R Elliott, Brady\u00a0T West, and Stephanie\u00a0M Coffey. 2023. An experimental evaluation of a stopping rule aimed at maximizing cost-quality trade-offs in surveys. Journal of the Royal Statistical Society Series A: Statistics in Society (2023), qnad059."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2013.80"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3447545.3451182"}],"event":{"name":"SC-W 2023: Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis","acronym":"SC-W 2023","location":"Denver CO USA"},"container-title":["Proceedings of the SC '23 Workshops of the International Conference on High Performance Computing, Network, Storage, and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3624062.3624202","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3624062.3624202","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T03:05:15Z","timestamp":1755745515000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3624062.3624202"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,12]]},"references-count":35,"alternative-id":["10.1145\/3624062.3624202","10.1145\/3624062"],"URL":"https:\/\/doi.org\/10.1145\/3624062.3624202","relation":{},"subject":[],"published":{"date-parts":[[2023,11,12]]},"assertion":[{"value":"2023-11-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}