{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T06:46:24Z","timestamp":1782197184330,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":63,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,10]]},"DOI":"10.1145\/3701717.3734462","type":"proceedings-article","created":{"date-parts":[[2025,7,16]],"date-time":"2025-07-16T13:52:43Z","timestamp":1752673963000},"page":"188-198","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["A Tale of Many Streams: Characterizing a Hybrid Batch-Stream Production Workload in Digazu, a Data Lake supported by Apache Kafka and Flink"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-4455-1985","authenticated-orcid":false,"given":"Donatien","family":"Schmitz","sequence":"first","affiliation":[{"name":"UCLouvain, Louvain-la-Neuve, Belgium"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-3232-461X","authenticated-orcid":false,"given":"Luc","family":"Berrewaerts","sequence":"additional","affiliation":[{"name":"Eura Nova, Mont-Saint-Guibert, Belgium"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8980-1231","authenticated-orcid":false,"given":"Guillaume","family":"Rosinosky","sequence":"additional","affiliation":[{"name":"INRIA, LS2N, UMR CNRS 6004, IMT Atlantique, F-44307 Nantes, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0664-5788","authenticated-orcid":false,"given":"Sabri","family":"Skhiri","sequence":"additional","affiliation":[{"name":"Eura Nova, Mont-Saint-Guibert, Belgium"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4133-394X","authenticated-orcid":false,"given":"Etienne","family":"Rivi\u00e8re","sequence":"additional","affiliation":[{"name":"UCLouvain, Louvain-la-Neuve, Belgium"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,6,9]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"2015. Yahoo Streaming Benchmark. https:\/\/developer.yahoo.com\/blogs\/135370591481\/?guccounter=1."},{"key":"e_1_3_3_2_3_2","unstructured":"2024. Aiven. https:\/\/aiven.io"},{"key":"e_1_3_3_2_4_2","unstructured":"2024. Kubernetes. https:\/\/kubernetes.io."},{"key":"e_1_3_3_2_5_2","unstructured":"2024. Prometheus. https:\/\/prometheus.io."},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE60146.2024.00163"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"crossref","unstructured":"Mutaz Barika Saurabh Garg Andrew Chan and Rodrigo\u00a0N Calheiros. 2019. Scheduling algorithms for efficient execution of stream workflow applications in multicloud environments. IEEE Transactions on Services Computing 15 2 (2019) 860\u2013875.","DOI":"10.1109\/TSC.2019.2963382"},{"key":"e_1_3_3_2_8_2","volume-title":"Kafka Streams in Action","author":"Bejeck Bill","year":"2024","unstructured":"Bill Bejeck. 2024. Kafka Streams in Action. Simon and Schuster."},{"key":"e_1_3_3_2_9_2","volume-title":"Conference on Innovative Data Systems Research (CIDR)","author":"Bernstein Philip\u00a0A","year":"2019","unstructured":"Philip\u00a0A Bernstein, Todd Porter, Rahul Potharaju, Alejandro\u00a0Z Tomsic, Shivaram Venkataraman, and Wentao Wu. 2019. Serverless Event-Stream Processing over Virtual Actors.. In Conference on Innovative Data Systems Research (CIDR)."},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Maycon\u00a0Viana Bordin Dalvan Griebler Gabriele Mencagli Cl\u00e1udio\u00a0FR Geyer and Luiz Gustavo\u00a0L Fernandes. 2020. DSPBench: A suite of benchmark applications for distributed data stream processing systems. IEEE Access 8 (2020) 222900\u2013222917.","DOI":"10.1109\/ACCESS.2020.3043948"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"crossref","unstructured":"Paris Carbone Stephan Ewen Gyula F\u00f3ra Seif Haridi Stefan Richter and Kostas Tzoumas. 2017. State management in Apache Flink\u00ae: consistent stateful distributed stream processing. Proceedings of the VLDB Endowment 10 12 (2017) 1718\u20131729.","DOI":"10.14778\/3137765.3137777"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3383131"},{"key":"e_1_3_3_2_13_2","unstructured":"Paris Carbone Asterios Katsifodimos Stephan Ewen Volker Markl Seif Haridi and Kostas Tzoumas. 2015. Apache Flink: Stream and batch processing in a single engine. The Bulletin of the Technical Committee on Data Engineering 38 4 (2015)."},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Valeria Cardellini Francesco Lo\u00a0Presti Matteo Nardelli and Gabriele\u00a0Russo Russo. 2022. Runtime Adaptation of Data Stream Processing Systems: The State of the Art. ACM Computing Surveys (CSUR) 54 11s (2022) 1\u201336.","DOI":"10.1145\/3514496"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"crossref","unstructured":"Yanpei Chen Sara Alspaugh and Randy Katz. 2012. Interactive Analytical Processing in Big Data Systems: A Cross-Industry Study of MapReduce Workloads. Proceedings of the VLDB Endowment 5 12 (2012).","DOI":"10.14778\/2367502.2367519"},{"key":"e_1_3_3_2_16_2","unstructured":"Hu\u00a0Zheng\u00a0(Alibaba Cloud). 2021. Building an Enterprise-Level Real-Time Data Lake Based on Flink and Iceberg. https:\/\/www.alibabacloud.com\/blog\/building-an-enterprise-level-real-time-data-lake-based-on-flink-and-iceberg_597755"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3514221.3517826"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2013.6691693"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"crossref","unstructured":"Siying Dong Andrew Kryczka Yanqin Jin and Michael Stumm. 2021. RocksDB: Evolution of development priorities in a key-value store serving large-scale applications. ACM Trans. on Storage 17 4 (2021).","DOI":"10.1145\/3483840"},{"key":"e_1_3_3_2_20_2","unstructured":"Allen\u00a0Wang (DoorDash). 2022. Building Scalable Real Time Event Processing with Kafka and Flink. https:\/\/doordash.engineering\/2022\/08\/02\/building-scalable-real-time-event-processing-with-kafka-and-flink\/"},{"key":"e_1_3_3_2_21_2","unstructured":"Apache Flink. 2024. Table API. https:\/\/nightlies.apache.org\/flink\/flink-docs-release-1.18\/docs\/dev\/table\/tableapi\/"},{"key":"e_1_3_3_2_22_2","unstructured":"Apache\u00a0Software Foundation. 2025. Apache Flink. https:\/\/flink.apache.org"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1109\/CLOUD49709.2020.00016"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"crossref","unstructured":"Stephen\u00a0R Gardner. 1998. Building the data warehouse. Commun. ACM 41 9 (1998) 52\u201360.","DOI":"10.1145\/285070.285080"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1109\/PERVASIVE.2015.7087172"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-82196-8_12"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Alon\u00a0Y Halevy Flip Korn Natalya\u00a0Fridman Noy Christopher Olston Neoklis Polyzotis Sudip Roy and Steven\u00a0Euijong Whang. 2016. Managing Google\u2019s data lake: an overview of the Goods system. IEEE Data Eng. Bull. 39 3 (2016) 5\u201314.","DOI":"10.1145\/2882903.2903730"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3427921.3450242"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"crossref","unstructured":"Martin Hirzel Robert Soul\u00e9 Scott Schneider Bu\u011fra Gedik and Robert Grimm. 2014. A catalog of stream processing optimizations. ACM Computing Surveys (CSUR) 46 4 (2014) 1\u201334.","DOI":"10.1145\/2528412"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Windsor\u00a0W Hsu Alan\u00a0Jay Smith and Honesty\u00a0C. Young. 2001. Characteristics of production database workloads and the TPC benchmarks. IBM Systems Journal 40 3 (2001) 781\u2013802.","DOI":"10.1147\/sj.403.0781"},{"key":"e_1_3_3_2_31_2","series-title":"(OSDI\u201918)","volume-title":"13th USENIX Symposium on Operating Systems Design and Implementation","author":"Kalavri Vasiliki","year":"2018","unstructured":"Vasiliki Kalavri, John Liagouris, Moritz Hoffmann, Desislava Dimitrova, Matthew Forshaw, and Timothy Roscoe. 2018. Three steps is all you need: fast, accurate, automatic scaling decisions for distributed streaming dataflows. In 13th USENIX Symposium on Operating Systems Design and Implementation(OSDI\u201918)."},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2018.00169"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3299869.3319884"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1109\/CICT.2013.6558077"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"crossref","unstructured":"Nikos\u00a0R Katsipoulakis Alexandros Labrinidis and Panos\u00a0K Chrysanthis. 2017. A holistic view of stream partitioning costs. Proceedings of the VLDB Endowment 10 11 (2017) 1286\u20131297.","DOI":"10.14778\/3137628.3137639"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.14778\/3625054.3625064"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"crossref","unstructured":"Branko Milanovic. 1997. A simple way to calculate the Gini coefficient and some implications. Economics Letters 56 1 (1997) 45\u201349.","DOI":"10.1016\/S0165-1765(97)00101-8"},{"key":"e_1_3_3_2_38_2","first-page":"1049","volume-title":"VLDB","author":"Nambiar Raghunath\u00a0Othayoth","year":"2006","unstructured":"Raghunath\u00a0Othayoth Nambiar and Meikel Poess. 2006. The Making of TPC-DS.. In VLDB , Vol.\u00a06. 1049\u20131058."},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"crossref","unstructured":"Fatemeh Nargesian Erkang Zhu Ren\u00e9e\u00a0J Miller Ken\u00a0Q Pu and Patricia\u00a0C Arocena. 2019. Data lake management: challenges and opportunities. Proceedings of the VLDB Endowment 12 12 (2019) 1986\u20131989.","DOI":"10.14778\/3352063.3352116"},{"key":"e_1_3_3_2_40_2","series-title":"(ICDE\u201916)","first-page":"589","volume-title":"2016 IEEE 32nd International Conference on Data Engineering","author":"Nasir Muhammad Anis\u00a0Uddin","year":"2016","unstructured":"Muhammad Anis\u00a0Uddin Nasir, Gianmarco De\u00a0Francisci Morales, Nicolas Kourtellis, and Marco Serafini. 2016. When two choices are not enough: Balancing at scale in distributed stream processing. In 2016 IEEE 32nd International Conference on Data Engineering(ICDE\u201916). 589\u2013600."},{"key":"e_1_3_3_2_41_2","unstructured":"Eura Nova. 2025. Digazu. https:\/\/digazu.com"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3464298.3493407"},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"crossref","unstructured":"Matthew Perron Raul Castro\u00a0Fernandez David DeWitt Michael Cafarella and Samuel Madden. 2023. Cackle: Analytical Workload Cost and Performance Stability With Elastic Pools. Proceedings of the ACM on Management of Data 1 4 (2023) 1\u201325.","DOI":"10.1145\/3626720"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"crossref","unstructured":"S\u00a0Yu Philip Hans-Ulrich Heiss Sukho Lee and Ming-Syan Chen. 1992. On Workload Characterization of Relational Database Environments. IEEE Trans. Software Eng. 18 4 (1992) 347\u2013355.","DOI":"10.1109\/32.129222"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2007.367885"},{"key":"e_1_3_3_2_46_2","first-page":"1138","volume-title":"VLDB","author":"Poess Meikel","year":"2007","unstructured":"Meikel Poess, Raghunath\u00a0Othayoth Nambiar, and David Walrath. 2007. Why You Should Run TPC-DS: A Workload Analysis.. In VLDB , Vol.\u00a07. 1138\u20131149."},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3056100"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"crossref","unstructured":"Theofanis\u00a0P Raptis and Andrea Passarella. 2023. A survey on networked data streaming with Apache Kafka. IEEE access (2023).","DOI":"10.1109\/ACCESS.2023.3303810"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-27615-7_23"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"crossref","unstructured":"Zujie Ren Jian Wan Weisong Shi Xianghua Xu and Min Zhou. 2013. Workload analysis implications and optimization on a production Hadoop cluster: A case study on TaoBao. IEEE Transactions on Services Computing 7 2 (2013) 307\u2013321.","DOI":"10.1109\/TSC.2013.40"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"crossref","unstructured":"Henriette R\u00f6ger and Ruben Mayer. 2019. A comprehensive survey on parallelization and elasticity in stream processing. ACM Computing Surveys (CSUR) 52 2 (2019) 1\u201337.","DOI":"10.1145\/3303849"},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"publisher","DOI":"10.1109\/SRDS53918.2021.00015"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3629104.3666034"},{"key":"e_1_3_3_2_54_2","first-page":"301","volume-title":"2023 USENIX Annual Technical Conference (USENIX ATC 23)","author":"Song Won\u00a0Wook","year":"2023","unstructured":"Won\u00a0Wook Song, Taegeon Um, Sameh Elnikety, Myeongjae Jeon, and Byung-Gon Chun. 2023. Sponge: Fast Reactive Scaling for Stream Processing with Serverless Frameworks. In 2023 USENIX Annual Technical Conference (USENIX ATC 23). 301\u2013314."},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"publisher","DOI":"10.1145\/3297663.3310302"},{"key":"e_1_3_3_2_56_2","volume-title":"NEXMark\u2013a benchmark for queries over data streams","author":"Tucker Pete","year":"2010","unstructured":"Pete Tucker, Kristin Tufte, Vassilis Papadimos, and David Maier. 2010. NEXMark\u2013a benchmark for queries over data streams. Technical Report. OGI School of Science & Engineering at OHSU."},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"crossref","unstructured":"Giselle Van\u00a0Dongen and Dirk Van\u00a0den Poel. 2020. Evaluation of stream processing frameworks. IEEE Transactions on Parallel and Distributed Systems 31 8 (2020) 1845\u20131858.","DOI":"10.1109\/TPDS.2020.2978480"},{"key":"e_1_3_3_2_58_2","doi-asserted-by":"crossref","unstructured":"Juliane Verwiebe Philipp\u00a0M Grulich Jonas Traub and Volker Markl. 2023. Survey of window types for aggregation in stream processing systems. The VLDB Journal (2023) 1\u201327.","DOI":"10.1007\/s00778-023-00793-1"},{"key":"e_1_3_3_2_59_2","doi-asserted-by":"crossref","unstructured":"Yuanli Wang Lei Huang Zikun Wang Vasiliki Kalavri and Ibrahim Matta. 2024. CAPSys: Contention-aware task placement for data stream processing(EuroSys).","DOI":"10.1145\/3689031.3696085"},{"key":"e_1_3_3_2_60_2","volume-title":"Hadoop: The definitive guide","author":"White Tom","year":"2012","unstructured":"Tom White. 2012. Hadoop: The definitive guide. \" O\u2019Reilly Media, Inc.\"."},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"crossref","unstructured":"Matei Zaharia Reynold\u00a0S Xin Patrick Wendell Tathagata Das Michael Armbrust Ankur Dave Xiangrui Meng Josh Rosen Shivaram Venkataraman Michael\u00a0J Franklin et\u00a0al. 2016. Apache Spark: a unified engine for big data processing. Commun. ACM 59 11 (2016) 56\u201365.","DOI":"10.1145\/2934664"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"crossref","unstructured":"Steffen Zeuch Bonaventura\u00a0Del Monte Jeyhun Karimov Clemens Lutz Manuel Renz Jonas Traub Sebastian Bre\u00df Tilmann Rabl and Volker Markl. 2019. Analyzing efficient stream processing on modern hardware. Proceedings of the VLDB Endowment 12 5 (2019) 516\u2013530.","DOI":"10.14778\/3303753.3303758"},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"crossref","unstructured":"Feng Zhang Chenyang Zhang Lin Yang Shuhao Zhang Bingsheng He Wei Lu and Xiaoyong Du. 2021. Fine-grained multi-query stream processing on integrated architectures. IEEE Transactions on Parallel and Distributed Systems 32 9 (2021) 2303\u20132320.","DOI":"10.1109\/TPDS.2021.3066407"},{"key":"e_1_3_3_2_64_2","series-title":"(HPCA\u201904)","first-page":"142","volume-title":"10th International Symposium on High Performance Computer Architecture","author":"Zhang Jianyong","year":"2004","unstructured":"Jianyong Zhang, Anand Sivasubramaniam, Hubertus Franke, Natarajan Gautam, Yanyong Zhang, and Shailabh Nagar. 2004. Synthesizing representative i\/o workloads for tpc-h. In 10th International Symposium on High Performance Computer Architecture(HPCA\u201904). IEEE, 142\u2013142."}],"event":{"name":"DEBS '25: The 19th ACM International Conference on Distributed and Event-based Systems","location":"Gothenburg Sweden","acronym":"DEBS '25","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 19th ACM International Conference on Distributed and Event-based Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3701717.3734462","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,16]],"date-time":"2025-07-16T13:54:08Z","timestamp":1752674048000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3701717.3734462"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,9]]},"references-count":63,"alternative-id":["10.1145\/3701717.3734462","10.1145\/3701717"],"URL":"https:\/\/doi.org\/10.1145\/3701717.3734462","relation":{},"subject":[],"published":{"date-parts":[[2025,6,9]]},"assertion":[{"value":"2025-06-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}