{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T04:32:53Z","timestamp":1782880373503,"version":"3.54.5"},"reference-count":34,"publisher":"MDPI AG","issue":"13","license":[{"start":{"date-parts":[[2022,6,23]],"date-time":"2022-06-23T00:00:00Z","timestamp":1655942400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"national funds through the \u201cSmartSent\u201d","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"national funds through the \u201cSmartSent\u201d","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"national funds through the \u201cSmartSent\u201d","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"national funds through the \u201cSmartSent\u201d","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"national funds through the \u201cSmartSent\u201d","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"national funds through the \u201cSmartSent\u201d","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]},{"name":"CAPES","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"CAPES","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"CAPES","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"CAPES","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"CAPES","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"CAPES","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]},{"name":"CEREIA Project","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"CEREIA Project","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"CEREIA Project","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"CEREIA Project","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"CEREIA Project","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"CEREIA Project","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"GREEN-CLOUD\u2014Computa\u00e7\u00e3o em Cloud com Computa\u00e7\u00e3o Sustent\u00e1vel","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"VALORIZA\u2013Research Centre for Endogenous Resource Valorization","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["#17\/2551-0001 195-3"],"award-info":[{"award-number":["#17\/2551-0001 195-3"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["Finance Code 001"],"award-info":[{"award-number":["Finance Code 001"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["#2020\/09706-7"],"award-info":[{"award-number":["#2020\/09706-7"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["#16\/2551-0000 488-9"],"award-info":[{"award-number":["#16\/2551-0000 488-9"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["UIDB\/05064\/2020"],"award-info":[{"award-number":["UIDB\/05064\/2020"]}]},{"name":"LIND\u2013Instituto Lus\u00f3fono de Investiga\u00e7\u00e3o e Desenvolvimento","award":["COFAC\/ILIND\/COPELABS\/3\/2020"],"award-info":[{"award-number":["COFAC\/ILIND\/COPELABS\/3\/2020"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Sensors"],"abstract":"<jats:p>A significant rise in the adoption of streaming applications has changed the decision-making processes in the last decade. This movement has led to the emergence of several Big Data technologies for in-memory processing, such as the systems Apache Storm, Spark, Heron, Samza, Flink, and others. Spark Streaming, a widespread open-source implementation, processes data-intensive applications that often require large amounts of memory. However, Spark Unified Memory Manager cannot properly manage sudden or intensive data surges and their related in-memory caching needs, resulting in performance and throughput degradation, high latency, a large number of garbage collection operations, out-of-memory issues, and data loss. This work presents a comprehensive performance evaluation of Spark Streaming backpressure to investigate the hypothesis that it could support data-intensive pipelines under specific pressure requirements. The results reveal that backpressure is suitable only for small and medium pipelines for stateless and stateful applications. Furthermore, it points out the Spark Streaming limitations that lead to in-memory-based issues for data-intensive pipelines and stateful applications. In addition, the work indicates potential solutions.<\/jats:p>","DOI":"10.3390\/s22134756","type":"journal-article","created":{"date-parts":[[2022,6,23]],"date-time":"2022-06-23T22:43:00Z","timestamp":1656024180000},"page":"4756","update-policy":"https:\/\/doi.org\/10.3390\/mdpi_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Performance Evaluation Analysis of Spark Streaming Backpressure for Data-Intensive Pipelines"],"prefix":"10.3390","volume":"22","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9131-6849","authenticated-orcid":false,"given":"Kassiano J.","family":"Matteussi","sequence":"first","affiliation":[{"name":"Institute of Informatics, Federal University of Rio Grande do Sul, UFRGS\/PPGC, Porto Alegre 91501-970, RS, Brazil"},{"name":"LIG-ERODS, Universit\u00e9 Grenoble Alpes, 38058 Grenoble, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3623-2762","authenticated-orcid":false,"given":"Julio C. S.","family":"dos Anjos","sequence":"additional","affiliation":[{"name":"Graduate Program in Teleinformatics Engineering Federal, University of Cear\u00e1, PPGETI\/UFC, Center of Technology, Campus of Pici, Fortaleza 60455-970, CE, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0446-9271","authenticated-orcid":false,"given":"Valderi R. Q.","family":"Leithardt","sequence":"additional","affiliation":[{"name":"COPELABS, Universidade Lus\u00f3fona de Humanidades e Tecnologias, 1749-024 Lisboa, Portugal"},{"name":"VALORIZA, Research Center for Endogenous Resource Valorization, Polytechnic Institute of Portalegre, 7300-555 Portalegre, Portugal"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8602-2336","authenticated-orcid":false,"given":"Claudio F. R.","family":"Geyer","sequence":"additional","affiliation":[{"name":"Institute of Informatics, Federal University of Rio Grande do Sul, UFRGS\/PPGC, Porto Alegre 91501-970, RS, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1968","published-online":{"date-parts":[[2022,6,23]]},"reference":[{"key":"ref_1","doi-asserted-by":"crossref","unstructured":"Hassanien, A.E., and Darwish, A. (2020). Machine Learning and Big Data Analytics Paradigms: Analysis, Applications and Challenges, Springer Nature.","DOI":"10.1007\/978-3-030-59338-4"},{"key":"ref_2","doi-asserted-by":"crossref","unstructured":"Avgeris, M., Spatharakis, D., Dechouniotis, D., Leivadeas, A., Karyotis, V., and Papavassiliou, S. (2022). ENERDGE: Distributed Energy-Aware Resource Allocation at the Edge. Sensors, 22.","DOI":"10.3390\/s22020660"},{"key":"ref_3","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1016\/j.jpdc.2020.03.010","article-title":"Dynamic Memory-Aware Scheduling in Spark Computing Environmen","volume":"141","author":"Tang","year":"2020","journal-title":"J. Parallel Distrib. Comput."},{"key":"ref_4","unstructured":"da Silva Veith, A., Dias de Assuncao, M., and Lefevre, L. (2021). Latency-Aware Strategies for Deploying Data Stream Processing Applications on Large Cloud-Edge Infrastructure. IEEE Trans. Cloud Comput., 11236."},{"key":"ref_5","doi-asserted-by":"crossref","unstructured":"Toshniwal, A., Taneja, S., Shukla, A., Ramasamy, K., Patel, J.M., Kulkarni, S., Jackson, J., Gade, K., Fu, M., and Donham, J. (2014, January 22\u201327). Storm@twitter. Proceedings of the ACM SIGMOD International Conference on Management of Data, Snowbird, UT, USA.","DOI":"10.1145\/2588555.2595641"},{"key":"ref_6","first-page":"1634","article-title":"Samza: Stateful Scalable Stream Processing at LinkedIn","volume":"10","author":"Noghabi","year":"2017","journal-title":"J. Very Large Data Base Endowment."},{"key":"ref_7","unstructured":"Zaharia, M., Chowdhury, M., Franklin, M.J., Shenker, S., and Stoica, I. (2010, January 22). Spark: Cluster Computing With Working Sets. Proceedings of the 2nd USENIX Workshop on Hot Topics in Cloud Computing (HotCloud 10), Boston, MA, USA."},{"key":"ref_8","first-page":"4","article-title":"Apache Flink: Stream and Batch Processing In A Single Engine","volume":"36","author":"Carbone","year":"2015","journal-title":"Bull. IEEE Comput. Soc. Tech. Comm. Data Eng."},{"key":"ref_9","unstructured":"Amazon Web Services, Inc. (2021, October 20). Collect Streaming Data, at Scale, for Real-Time Analytics. Available online: https:\/\/aws.amazon.com\/kinesis\/data-streams\/."},{"key":"ref_10","doi-asserted-by":"crossref","unstructured":"Xu, L., Li, M., Zhang, L., Butt, A.R., Wang, Y., and Hu, Z.Z. (2016, January 3\u201327). MEMTUNE: Dynamic Memory Management for In-Memory Data Analytic Platforms. Proceedings of the IEEE International Parallel and Distributed Processing Symposium (IPDPS), Chicago, IL, USA.","DOI":"10.1109\/IPDPS.2016.105"},{"key":"ref_11","doi-asserted-by":"crossref","unstructured":"Zhao, Z., Zhang, H., Geng, X., and Ma, H. (2019, January 16\u201318). Resource-Aware Cache Management for In-Memory Data Analytics Frameworks. Proceedings of the IEEE Intl Conf on Parallel Distributed Processing with Applications, Big Data Cloud Computing, Sustainable Computing Communications, Social Computing Networking (ISPA\/BDCloud\/SocialCom\/SustainCom), Xiamen, China.","DOI":"10.1109\/ISPA-BDCloud-SustainCom-SocialCom48970.2019.00060"},{"key":"ref_12","doi-asserted-by":"crossref","unstructured":"Jia, D., Bhimani, J., Nguyen, S.N., Sheng, B., and Mi, N. (2019, January 29\u201331). Atumm: Auto-Tuning Memory Manager in Apache Spark. Proceedings of the IEEE International Conference on Performance, Computing and Communications (IPCCC), London, UK.","DOI":"10.1109\/IPCCC47392.2019.8958724"},{"key":"ref_13","unstructured":"Matteussi, K.J., Zanchetta, B.F., Bertoncello, G., Dos Santos, J.D., Dos Anjos, J.C., and Geyer, C.F. (July, January 29). Analysis and Performance Evaluation of Deep Learning on Big Data. Proceedings of the IEEE Symposium on Computers and Communications (ISCC), Barcelona, Spain."},{"key":"ref_14","doi-asserted-by":"crossref","unstructured":"Lopes, H., Pires, I.M., S\u00e1nchez San Blas, H., Garc\u00eda-Ovejero, R., and Leithardt, V. (2020). PriADA: Management and Adaptation of Information Based on Data Privacy in Public Environments. Computers, 9.","DOI":"10.3390\/computers9040077"},{"key":"ref_15","doi-asserted-by":"crossref","first-page":"219124","DOI":"10.1109\/ACCESS.2020.3042739","article-title":"Boosting Big Data Streaming Applications in Clouds With BurstFlow","volume":"8","author":"Matteussi","year":"2020","journal-title":"IEEE Access"},{"key":"ref_16","doi-asserted-by":"crossref","unstructured":"Matteussi, K.J., Geyer, C.F.R., Xavier, M.G., and De Rose, C.A. (2018, January 16\u201320). Understanding and Minimizing Disk Contention Effects for Data-Intensive Processing in Virtualized Systems. Proceedings of the Proceedings of International Conference on High Performance Computing Simulation (HPCS). IEEE Computer Society, Orleans, France.","DOI":"10.1109\/HPCS.2018.00144"},{"key":"ref_17","doi-asserted-by":"crossref","first-page":"170281","DOI":"10.1109\/ACCESS.2020.3023344","article-title":"Data Processing Model to Perform Big Data Analytics in Hybrid Infrastructures","volume":"8","author":"Matteussi","year":"2020","journal-title":"IEEE Access"},{"key":"ref_18","doi-asserted-by":"crossref","unstructured":"Dos Anjos, J.C.S., Gross, J.L.G., Matteussi, K.J., Gonz\u00e1lez, G.V., Leithardt, V.R.Q., and Geyer, C.F.R. (2021). An Algorithm to Minimize Energy Consumption and Elapsed Time for IoT Workloads in a Hybrid Architecture. Sensors, 21.","DOI":"10.3390\/s21092914"},{"key":"ref_19","doi-asserted-by":"crossref","first-page":"100895","DOI":"10.1016\/j.softx.2021.100895","article-title":"PADRES: Tool for PrivAcy, Data REgulation and Security","volume":"17","author":"Pereira","year":"2022","journal-title":"SoftwareX"},{"key":"ref_20","doi-asserted-by":"crossref","unstructured":"Chen, L., Zhao, J., Wang, C., Cao, T., Zigman, J., Volos, H., Mutlu, O., Lv, F., Feng, X., and Xu, G.H. Unified Holistic Memory Management Supporting Multiple Big Data Processing Frameworks over Hybrid Memories. ACM Trans. Comput. Syst., 2022.","DOI":"10.1145\/3511211"},{"key":"ref_21","doi-asserted-by":"crossref","unstructured":"Hanif, M., Yoon, H., and Lee, C. (2020, January 7\u201310). A Backpressure Mitigation Scheme in Distributed Stream Processing Engines. Proceedings of the 2020 International Conference on Information Networking (ICOIN), Barcelona, Spain.","DOI":"10.1109\/ICOIN48656.2020.9016513"},{"key":"ref_22","doi-asserted-by":"crossref","unstructured":"Das, T., Zhong, Y., Stoica, I., and Shenker, S. (2014, January 3\u20135). Adaptive Stream Processing Using Dynamic Batch Sizing. Proceedings of the ACM Symposium on Cloud Computing, Seattle, WA, USA.","DOI":"10.1145\/2670979.2670995"},{"key":"ref_23","doi-asserted-by":"crossref","unstructured":"Birke, R., Bj\u00f6erkqvist, M., Kalyvianaki, E., and Chen, L.Y. (2017, January 4\u20137). Meeting Latency Target in Transient Burst: A Case on Spark Streaming. Proceedings of the 2017 IEEE International Conference on Cloud Engineering (IC2E), Vancouver, BC, Canada.","DOI":"10.1109\/IC2E.2017.17"},{"key":"ref_24","doi-asserted-by":"crossref","unstructured":"Chen, X., Vigfusson, Y., Blough, D.M., Zheng, F., Wu, K.L., and Hu, L. (2017, January 17\u201321). GOVERNOR: Smoother Stream Processing Through Smarter Backpressure. Proceedings of the IEEE International Conference on Autonomic Computing (ICAC), Columbus, OH, USA.","DOI":"10.1109\/ICAC.2017.31"},{"key":"ref_25","doi-asserted-by":"crossref","first-page":"220","DOI":"10.1115\/1.2899060","article-title":"Optimum Settings for Automatic Controllers","volume":"115","author":"Ziegler","year":"1993","journal-title":"J. Dyn. Syst. Meas. Control"},{"key":"ref_26","unstructured":"Startin, R. (2021, May 17). Tuning Spark Back Pressure by Simulation. Available online: https:\/\/richardstartin.github.io\/posts\/tuning-spark-back-pressure-by-simulation."},{"key":"ref_27","doi-asserted-by":"crossref","unstructured":"Zaharia, M., Das, T., Li, H., Hunter, T., Shenker, S., and Stoica, I. (2013, January 3\u20136). Discretized Streams: Fault-Tolerant Streaming Computation at Scale. Proceedings of the Twenty-Fourth ACM Symposium on Operating Systems Principles, Farminton, PA, USA.","DOI":"10.1145\/2517349.2522737"},{"key":"ref_28","doi-asserted-by":"crossref","unstructured":"Dessokey, M., Saif, S.M., Salem, S., Saad, E., and Eldeeb, H. (2020, January 19\u201321). Memory Management Approaches in Apache Spark: A Review. Proceedings of the International Conference on Advanced Intelligent Systems and Informatics 2020, Cairo, Egypt.","DOI":"10.1007\/978-3-030-58669-0_36"},{"key":"ref_29","unstructured":"Or, A., and Rosen, J. (2021, September 23). Unified Memory Management in Spark 1.6. Available online: https:\/\/www.linuxprobe.com\/wp-content\/uploads\/2017\/04\/unified-memory-management-spark-10000.pdf."},{"key":"ref_30","unstructured":"Daoyuan, W., and Huang, J. (2022, March 05). Tuning Java Garbage Collection for Apache Spark Applications. Available online: https:\/\/databricks.com\/blog\/2015\/05\/28\/tuning-java-garbage-collection-for-spark-applications.html."},{"key":"ref_31","unstructured":"(2022, May 05). Apache Spark. Hardware Provisioning. Available online: https:\/\/spark.apache.org\/docs\/3.0.0\/hardware-provisioning.html."},{"key":"ref_32","unstructured":"(2021, September 07). Apache Spark. Memory Management Overview. Available online: https:\/\/spark.apache.org\/docs\/latest\/tuning.html#memory-management-overview."},{"key":"ref_33","unstructured":"(2021, November 17). \u00d8MQ\u2014The Guide. Available online: http:\/\/zguide.zeromq.org\/php:chapter2."},{"key":"ref_34","unstructured":"DataFlair (2021, February 15). Apache Spark DStream. Available online: https:\/\/data-flair.training\/blogs\/spark-tutorial\/."}],"container-title":["Sensors"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.mdpi.com\/1424-8220\/22\/13\/4756\/pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T23:38:53Z","timestamp":1760139533000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.mdpi.com\/1424-8220\/22\/13\/4756"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,23]]},"references-count":34,"journal-issue":{"issue":"13","published-online":{"date-parts":[[2022,7]]}},"alternative-id":["s22134756"],"URL":"https:\/\/doi.org\/10.3390\/s22134756","relation":{"has-preprint":[{"id-type":"doi","id":"10.20944\/preprints202205.0334.v1","asserted-by":"object"}]},"ISSN":["1424-8220"],"issn-type":[{"value":"1424-8220","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,6,23]]}}}