{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T20:28:15Z","timestamp":1793219295946,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3696630.3728563","type":"proceedings-article","created":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:09:27Z","timestamp":1753729767000},"page":"400-410","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["When Should I Run My Application Benchmark?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6912-2549","authenticated-orcid":false,"given":"S\u00f6ren","family":"Henning","sequence":"first","affiliation":[{"name":"Dynatrace Research, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3299-2641","authenticated-orcid":false,"given":"Adriano","family":"Vogel","sequence":"additional","affiliation":[{"name":"Dynatrace Research, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3572-686X","authenticated-orcid":false,"given":"Esteban","family":"Perez-Wohlfeil","sequence":"additional","affiliation":[{"name":"Dynatrace Research, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7322-6332","authenticated-orcid":false,"given":"Otmar","family":"Ertl","sequence":"additional","affiliation":[{"name":"Dynatrace Research, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3862-1112","authenticated-orcid":false,"given":"Rick","family":"Rabiser","sequence":"additional","affiliation":[{"name":"LIT CPS Lab, Johannes Kepler University Linz, Linz, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 8th ACM\/SPEC on International Conference on Performance Engineering (L'Aquila, Italy) (ICPE '17)","author":"Abedi Ali","year":"2017","unstructured":"Ali Abedi and Tim Brecht. 2017. Conducting Repeatable Experiments in Highly Variable Cloud Computing Environments. In Proceedings of the 8th ACM\/SPEC on International Conference on Performance Engineering (L'Aquila, Italy) (ICPE '17). ACM, 287\u2013292. 10.1145\/3030207.3030229"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1002\/spe.3244"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-55483-9"},{"key":"e_1_3_2_1_4_1","volume-title":"2025 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). IEEE. In press. 10","author":"Borysenkov Dmytro","year":"2025","unstructured":"Dmytro Borysenkov, Adriano Vogel, S\u00f6ren Henning, and Esteban Perez-Wohlfeil. 2025. Analyzing Logs of Large-Scale Software Systems using Time Curves Visualization. In 2025 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). IEEE. In press. 10.1109\/SANER64311.2025.00038"},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of the 8th ACM\/SPEC on International Conference on Performance Engineering Companion (L'Aquila, Italy) (ICPE '17 Companion). ACM, 185\u2013189","author":"Bulej Lubom\u00edr","year":"2017","unstructured":"Lubom\u00edr Bulej, Vojtech Hork\u00fd, and Petr T\u016fma. 2017. Do We Teach Useful Statistics for Performance Evaluation?. In Proceedings of the 8th ACM\/SPEC on International Conference on Performance Engineering Companion (L'Aquila, Italy) (ICPE '17 Companion). ACM, 185\u2013189. 10.1145\/3053600.3053638"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the ACM\/SPEC International Conference on Performance Engineering (Edmonton AB, Canada) (ICPE '20)","author":"Bulej Lubom\u00edr","year":"2020","unstructured":"Lubom\u00edr Bulej, Vojt\u011bch Hork\u00fd, Petr Tuma, Fran\u00e7ois Farquet, and Aleksandar Prokopec. 2020. Duet Benchmarking: Improving Measurement Accuracy in the Cloud. In Proceedings of the ACM\/SPEC International Conference on Performance Engineering (Edmonton AB, Canada) (ICPE '20). ACM, 100\u2013107. 10.1145\/3358960.3379132"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111294"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.14778\/3654621.3654632"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-36727-4_12"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","first-page":"24","DOI":"10.1109\/MWC.2013.6549280","article-title":"Migrate or not? Exploiting dynamic task migration in mobile cloud computing systems","volume":"20","author":"Gkatzikis Lazaros","year":"2013","unstructured":"Lazaros Gkatzikis and Iordanis Koutsopoulos. 2013. Migrate or not? Exploiting dynamic task migration in mobile cloud computing systems. IEEE Wireless Communications 20, 3 (2013), 24\u201332.","journal-title":"IEEE Wireless Communications"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3463274.3463361"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678687"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering (Tallinn, Estonia) (ESEC\/FSE","author":"He Sen","year":"2019","unstructured":"Sen He, Glenna Manns, John Saunders, Wei Wang, Lori Pollock, and Mary Lou Soffa. 2019. A Statistics-Based Performance Testing Methodology for Cloud Applications. In Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering (Tallinn, Estonia) (ESEC\/FSE 2019). ACM, 188\u2013199. 10.1145\/3338906.3338912"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-022-10162-1"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111879"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 15th ACM\/SPEC International Conference on Performance Engineering","author":"Henning S\u00f6ren","year":"2024","unstructured":"S\u00f6ren Henning, Adriano Vogel, Michael Leichtfried, Otmar Ertl, and Rick Rabiser. 2024. ShuffleBench: A Benchmark for Large-Scale Data Shuffling Operations with Distributed Stream Processing Frameworks. In Proceedings of the 15th ACM\/SPEC International Conference on Performance Engineering (London, United Kingdom) (ICPE '24). ACM, 2\u201313. 10.1145\/3629526.3645036"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","unstructured":"S\u00f6ren Henning Adriano Vogel Esteban Perez-Wohlfeil Otmar Ertl and Rick Rabiser. 2025. Replication Package for: When Should I Run My Application Benchmark?: Studying Cloud Performance Variability for the Case of Stream Processing Applications. 10.5281\/zenodo.14617187","DOI":"10.5281\/zenodo.14617187"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the ACM\/SPEC International Conference on Performance Engineering (ICPE '21)","author":"Hesse Guenter","year":"2021","unstructured":"Guenter Hesse, Christoph Matthies, Michael Perscheid, Matthias Uflacker, and Hasso Plattner. 2021. ESPBench: The Enterprise Stream Processing Benchmark. In Proceedings of the ACM\/SPEC International Conference on Performance Engineering (ICPE '21). ACM, 201\u2013212. 10.1145\/3427921.3450242"},{"key":"e_1_3_2_1_19_1","volume-title":"On the Performance Variability of Production Cloud Services. In 2011 11th IEEE\/ACM International Symposium on Cluster, Cloud and Grid Computing. IEEE, 104\u2013113","author":"Iosup Alexandru","year":"2011","unstructured":"Alexandru Iosup, Nezih Yigitbasi, and Dick Epema. 2011. On the Performance Variability of Production Cloud Services. In 2011 11th IEEE\/ACM International Symposium on Cluster, Cloud and Grid Computing. IEEE, 104\u2013113. 10.1109\/CCGrid.2011.22"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-41705-5"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09681-1"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2885497"},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 13th USENIX Conference on Operating Systems Design and Implementation","author":"Maricq Aleksander","year":"2018","unstructured":"Aleksander Maricq, Dmitry Duplyakin, Ivo Jimenez, Carlos Maltzahn, Ryan Stutsman, and Robert Ricci. 2018. Taming Performance Variability. In Proceedings of the 13th USENIX Conference on Operating Systems Design and Implementation (Carlsbad, CA, USA) (OSDI'18). USENIX Association, 409\u2013425."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3150224"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2927908"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","unstructured":"Julian Reichinger Thomas Krismayer and Jan Rellermeyer. 2024. COPR - Efficient large-scale log storage and retrieval. arXiv:2402.18355 [cs.IR] 10.48550\/arXiv.2402.18355","DOI":"10.48550\/arXiv.2402.18355"},{"key":"e_1_3_2_1_27_1","first-page":"10","volume-title":"Proceedings of the International Workshop on Real-Time Business Intelligence and Analytics (BIRTE '18)","author":"Sax Matthias J.","year":"2018","unstructured":"Matthias J. Sax, Guozhang Wang, Matthias Weidlich, and Johann-Christoph Freytag. 2018. Streams and Tables: Two Sides of the Same Coin. In Proceedings of the International Workshop on Real-Time Business Intelligence and Analytics (BIRTE '18). ACM, 10 pages. 10.1145\/3242153.3242155"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.14778\/1920841.1920902"},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the 1st Workshop on SErverless Systems, Applications and MEthodologies","author":"Schirmer Trever","year":"2023","unstructured":"Trever Schirmer, Nils Japke, Sofia Greten, Tobias Pfandzelter, and David Bermbach. 2023. The Night Shift: Understanding Performance Variability of Cloud Serverless Platforms. In Proceedings of the 1st Workshop on SErverless Systems, Applications and MEthodologies (Rome, Italy) (SESAME '23). ACM, 27\u201333. 10.1145\/3592533.3592808"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3241743"},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the 17th Usenix Conference on Networked Systems Design and Implementation","author":"Uta Alexandru","year":"2020","unstructured":"Alexandru Uta, Alexandru Custura, Dmitry Duplyakin, Ivo Jimenez, Jan Rellermeyer, Carlos Maltzahn, Robert Ricci, and Alexandru Iosup. 2020. Is big data performance reproducible in modern cloud networks?. In Proceedings of the 17th Usenix Conference on Networked Systems Design and Implementation (Santa Clara, CA, USA) (NSDI'20). USENIX Association, 513\u2013528."},{"key":"e_1_3_2_1_32_1","volume-title":"Euromicro Conference on Software Engineering and Advanced Applications. IEEE. 10","author":"Vogel Adriano","year":"2023","unstructured":"Adriano Vogel, S\u00f6ren Henning, Otmar Ertl, and Rick Rabiser. 2023. A systematic mapping of performance in distributed stream processing systems. In Euromicro Conference on Software Engineering and Advanced Applications. IEEE. 10.1109\/SEAA60479.2023.00052"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the 18th ACM International Conference on Distributed and Event-Based Systems","author":"Vogel Adriano","year":"2024","unstructured":"Adriano Vogel, S\u00f6ren Henning, Esteban Perez-Wohlfeil, Otmar Ertl, and Rick Rabiser. 2024. A Comprehensive Benchmarking Analysis of Fault Recovery in Stream Processing Frameworks. In Proceedings of the 18th ACM International Conference on Distributed and Event-Based Systems (Villeurbanne, France) (DEBS '24). ACM, 171\u2013182. 10.1145\/3629104.3666040"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 2021 International Conference on Management of Data (SIGMOD\/PODS '21)","author":"Wang Guozhang","year":"2021","unstructured":"Guozhang Wang, Lei Chen, Ayusman Dikshit, Jason Gustafson, Boyang Chen, Matthias J. Sax, John Roesler, Sophie Blee-Goldman, Bruno Cadonna, Apurva Mehta, Varun Madan, and Jun Rao. 2021. Consistency and Completeness: Rethinking Distributed Stream Processing in Apache Kafka. In Proceedings of the 2021 International Conference on Management of Data (SIGMOD\/PODS '21). ACM, 2602\u20132613. 10.1145\/3448016.3457556"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-025-10615-3"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111395"},{"key":"e_1_3_2_1_37_1","volume-title":"Cloud Performance Variability Prediction. In Companion of the ACM\/SPEC International Conference on Performance Engineering","author":"Zhao Yuxuan","year":"2021","unstructured":"Yuxuan Zhao, Dmitry Duplyakin, Robert Ricci, and Alexandru Uta. 2021. Cloud Performance Variability Prediction. In Companion of the ACM\/SPEC International Conference on Performance Engineering (Virtual Event, France) (ICPE '21). ACM, 35\u201340. 10.1145\/3447545.3451182"}],"event":{"name":"FSE Companion '25: 33rd ACM International Conference on the Foundations of Software Engineering","location":"Clarion Hotel Trondheim Trondheim Norway","acronym":"FSE Companion '25","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3696630.3728563","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:13:40Z","timestamp":1753730020000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696630.3728563"}},"subtitle":["Studying Cloud Performance Variability for the Case of Stream Processing Applications"],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":37,"alternative-id":["10.1145\/3696630.3728563","10.1145\/3696630"],"URL":"https:\/\/doi.org\/10.1145\/3696630.3728563","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-07-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}