{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,30]],"date-time":"2026-07-30T14:14:26Z","timestamp":1785420866701,"version":"3.56.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2025,6,16]],"date-time":"2025-06-16T00:00:00Z","timestamp":1750032000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2025,6,16]],"date-time":"2025-06-16T00:00:00Z","timestamp":1750032000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"Umea University"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Cluster Comput"],"published-print":{"date-parts":[[2025,9]]},"abstract":"<jats:title>Abstract<\/jats:title>\n          <jats:p>Efficient asset management is essential for optimizing the performance and scalability of modern Big Data (BD) frameworks. However, traditional resource allocation methods often suffer from static partitioning, inefficient resource utilization, and high operational costs, limiting their ability to adapt to fluctuating workloads dynamically. This paper introduces BigOPERA, an opportunistic and elastic resource allocation framework designed to enhance BD processing environments by integrating dedicated and non-dedicated computing assets. Leveraging containerization and a two-tiered scheduling mechanism, BigOPERA dynamically manages available resources to improve workload execution efficiency. Experimental results demonstrate that BigOPERA achieves up to 35% performance improvement over native Apache Spark configurations, significantly enhancing computational throughput while optimizing resource consumption. Our findings highlight the potential of BigOPERA in scalable, cost-effective, and sustainable BD processing.<\/jats:p>","DOI":"10.1007\/s10586-025-05274-4","type":"journal-article","created":{"date-parts":[[2025,6,16]],"date-time":"2025-06-16T12:19:55Z","timestamp":1750076395000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["BigOPERA: An OPportunistic and Elastic Resource Allocation for big data frameworks"],"prefix":"10.1007","volume":"28","author":[{"given":"Pablo V.","family":"Caderno","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Feras","family":"Awaysheh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jos\u00e9 C.","family":"Cabaleiro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tom\u00e1s F.","family":"Pena","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,6,16]]},"reference":[{"key":"5274_CR1","doi-asserted-by":"publisher","first-page":"43","DOI":"10.30564\/jcsr.v5i4.6072","volume":"5","author":"Z Sun","year":"2023","unstructured":"Sun, Z.: Data, analytics, and intelligence. J. Comput. Sci. Res. 5, 43\u201357 (2023). https:\/\/doi.org\/10.30564\/jcsr.v5i4.6072","journal-title":"J. Comput. Sci. Res."},{"key":"5274_CR2","doi-asserted-by":"publisher","first-page":"726","DOI":"10.1016\/j.future.2020.02.052","volume":"108","author":"FM Awaysheh","year":"2020","unstructured":"Awaysheh, F.M., Alazab, M., Gupta, M., Pena, T.F., Cabaleiro, J.C.: Next-generation big data federation access control: a reference model. Future Gener. Comput. Syst. 108, 726\u2013741 (2020)","journal-title":"Future Gener. Comput. Syst."},{"issue":"1","key":"5274_CR3","doi-asserted-by":"publisher","first-page":"4539","DOI":"10.1038\/s41598-024-55061-1","volume":"14","author":"HN Fakhouri","year":"2024","unstructured":"Fakhouri, H.N., Alawadi, S., Awaysheh, F.M., Alkhabbas, F., Zraqou, J.: A cognitive deep learning approach for medical image processing. Sci. Rep. 14(1), 4539 (2024)","journal-title":"Sci. Rep."},{"issue":"7","key":"5274_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s00607-024-01287-w","volume":"106","author":"HN Fakhouri","year":"2024","unstructured":"Fakhouri, H.N., Awaysheh, F.M., Alawadi, S., Alkhalaileh, M., Hamad, F.: Four vector intelligent metaheuristic for data optimization. Computing 106(7), 1\u201339 (2024)","journal-title":"Computing"},{"issue":"6","key":"5274_CR5","doi-asserted-by":"publisher","first-page":"5009","DOI":"10.22178\/pos.47-6","volume":"5","author":"R Nda","year":"2019","unstructured":"Nda, R., Tasmin, R.: Big data management in education sector: an overview. Path Sci. 5(6), 5009\u20135014 (2019). https:\/\/doi.org\/10.22178\/pos.47-6","journal-title":"Path Sci."},{"issue":"4","key":"5274_CR6","doi-asserted-by":"publisher","first-page":"2098","DOI":"10.1109\/COMST.2021.3094993","volume":"23","author":"FM Awaysheh","year":"2021","unstructured":"Awaysheh, F.M., Alazab, M., Garg, S., Niyato, D., Verikoukis, C.: Big data resource management and networks: taxonomy, survey, and future directions. IEEE Commun. Surv. Tutor. 23(4), 2098\u20132130 (2021)","journal-title":"IEEE Commun. Surv. Tutor."},{"key":"5274_CR7","doi-asserted-by":"publisher","first-page":"191","DOI":"10.1016\/j.jpdc.2019.09.015","volume":"135","author":"M Xu","year":"2020","unstructured":"Xu, M., Buyya, R.: Managing renewable energy and carbon footprint in multi-cloud computing environments. J. Parallel Distrib. Comput. 135, 191\u2013202 (2020). https:\/\/doi.org\/10.1016\/j.jpdc.2019.09.015","journal-title":"J. Parallel Distrib. Comput."},{"issue":"6","key":"5274_CR8","doi-asserted-by":"publisher","first-page":"3676","DOI":"10.1109\/TEM.2020.3045661","volume":"69","author":"FM Awaysheh","year":"2021","unstructured":"Awaysheh, F.M., Aladwan, M.N., Alazab, M., Alawadi, S., Cabaleiro, J.C., Pena, T.F.: Security by design for big data frameworks over cloud computing. IEEE Trans. Eng. Manag. 69(6), 3676\u20133693 (2021)","journal-title":"IEEE Trans. Eng. Manag."},{"key":"5274_CR9","doi-asserted-by":"crossref","first-page":"1001","DOI":"10.52783\/tjjpt.v44.i2.165","volume":"44","author":"N Kumar","year":"2023","unstructured":"Kumar, N., Hema, K., Sai, S., Hordiichuk, V., Menon, R., Catherene, D., Aarthy, J., Gonesh, C., Saha, G., Balaji, K.: Harnessing the power of big data: challenges and opportunities in analytics. Tuijin Jishu\/J. Propuls. Technol. 44, 1001\u20134055 (2023)","journal-title":"Tuijin Jishu\/J. Propuls. Technol."},{"key":"5274_CR10","unstructured":"Apache Software Foundation: Apache Hadoop. Apache Software Foundation. https:\/\/hadoop.apache.org. Accessed 22 Oct 2024"},{"key":"5274_CR11","unstructured":"Zaharia, M., Chowdhury, M., Franklin, M.J., Shenker, S., Stoica, I.: Spark: cluster computing with working sets. In: Proceedings of the 2nd USENIX Conference on Hot Topics in Cloud Computing, HotCloud\u201910, 2010, p. 10. USENIX Association (2010)"},{"key":"5274_CR12","doi-asserted-by":"publisher","unstructured":"Lin, H., Ma, X., Archuleta, J., Feng, W.C., Gardner, M., Zhang, Z.: MOON: MapReduce on opportunistic environments. In: Proceedings of the 19th ACM International Symposium on High Performance Distributed Computing, HPDC \u201910, 2010, pp. 95\u2013106. Association for Computing Machinery, New York (2010). https:\/\/doi.org\/10.1145\/1851476.1851489","DOI":"10.1145\/1851476.1851489"},{"key":"5274_CR13","doi-asserted-by":"publisher","unstructured":"Yao, Y., Gao, H., Wang, J., Mi, N., Sheng, B.: OpERA: opportunistic and efficient resource allocation in Hadoop YARN by harnessing idle resources. In: 2016 25th International Conference on Computer Communication and Networks (ICCCN), 2016, pp. 1\u20139 (2016). https:\/\/doi.org\/10.1109\/ICCCN.2016.7568553","DOI":"10.1109\/ICCCN.2016.7568553"},{"key":"5274_CR14","unstructured":"Zaharia, M., Konwinski, A., Joseph, A.D., Katz, R., Stoica, I.: Improving MapReduce performance in heterogeneous environments. In: Proceedings of the 8th USENIX Conference on Operating Systems Design and Implementation, OSDI\u201908, 2008, pp. 29\u201342. USENIX Association (2008)"},{"issue":"1","key":"5274_CR15","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1145\/1327452.1327492","volume":"51","author":"J Dean","year":"2008","unstructured":"Dean, J., Ghemawat, S.: MapReduce: simplified data processing on large clusters. Commun. ACM 51(1), 107\u2013113 (2008)","journal-title":"Commun. ACM"},{"key":"5274_CR16","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.jnca.2014.07.022","volume":"46","author":"I Polato","year":"2014","unstructured":"Polato, I., R\u00e9, R., Goldman, A., Kon, F.: A comprehensive view of Hadoop research\u2014a systematic literature review. J. Netw. Comput. Appl. 46, 1\u201325 (2014). https:\/\/doi.org\/10.1016\/j.jnca.2014.07.022","journal-title":"J. Netw. Comput. Appl."},{"key":"5274_CR17","unstructured":"Bawankule, K., Kumar Singh, A., Kumar Dewaang, R.: Analysis of task scheduling in Hadoop MapReduce framework. Aust. J. Wirel. Technol. Mobil. Secur. 1 (2022). https:\/\/ausjournal.com\/index.php\/j\/article\/view\/19"},{"key":"5274_CR18","doi-asserted-by":"crossref","unstructured":"Ragab, M., Awaysheh, F.M., Tommasini, R.: Bench-Ranking: a first step towards prescriptive performance analyses for big data frameworks. In: 2021 IEEE International Conference on Big Data (Big Data), 2021, pp. 241\u2013251. IEEE (2021)","DOI":"10.1109\/BigData52589.2021.9671277"},{"key":"5274_CR19","doi-asserted-by":"publisher","DOI":"10.1016\/j.jnca.2022.103405","volume":"204","author":"T Khan","year":"2022","unstructured":"Khan, T., Tian, W., Zhou, G., Ilager, S., Gong, M., Buyya, R.: Machine learning (ML)-centric resource management in cloud computing: a review and future directions. J. Netw. Comput. Appl. 204, 103405 (2022)","journal-title":"J. Netw. Comput. Appl."},{"key":"5274_CR20","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1016\/j.future.2021.10.019","volume":"128","author":"T Khan","year":"2022","unstructured":"Khan, T., Tian, W., Ilager, S., Buyya, R.: Workload forecasting and energy state estimation in cloud data centres: ML-centric approach. Future Gener. Comput. Syst. 128, 320\u2013332 (2022)","journal-title":"Future Gener. Comput. Syst."},{"key":"5274_CR21","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1007\/s10723-015-9344-9","volume":"13","author":"A Verma","year":"2015","unstructured":"Verma, A., Kaushal, S.: Cost-time efficient scheduling plan for executing workflows in the cloud. J. Grid Comput. 13, 495\u2013506 (2015). https:\/\/doi.org\/10.1007\/s10723-015-9344-9","journal-title":"J. Grid Comput."},{"key":"5274_CR22","unstructured":"Hindman, B., Konwinski, A., Zaharia, M., Ghodsi, A., Joseph, A.D., Katz, R., Shenker, S., Stoica, I.: Mesos: a platform for fine-grained resource sharing in the data center. In: Proceedings of the 8th USENIX Conference on Networked Systems Design and Implementation, NSDI\u201911, 2011, pp. 295\u2013308. USENIX Association (2011)"},{"issue":"3","key":"5274_CR23","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1109\/MCC.2014.51","volume":"1","author":"D Bernstein","year":"2014","unstructured":"Bernstein, D.: Containers and cloud: from LXC to Docker to Kubernetes. IEEE Cloud Comput. 1(3), 81\u201384 (2014). https:\/\/doi.org\/10.1109\/MCC.2014.51","journal-title":"IEEE Cloud Comput."},{"key":"5274_CR24","doi-asserted-by":"publisher","unstructured":"Ousterhout, K., Wendell, P., Zaharia, M., Stoica, I.: Sparrow: distributed, low latency scheduling. In: Proceedings of the Twenty-Fourth ACM Symposium on Operating Systems Principles, SOSP\u201913, 2013, pp. 69\u201384. Association for Computing Machinery, New York (2013). https:\/\/doi.org\/10.1145\/2517349.2522716","DOI":"10.1145\/2517349.2522716"},{"key":"5274_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TMC.2025.3540407","volume":"1","author":"Z Chen","year":"2025","unstructured":"Chen, Z., Huang, S., Min, G., Ning, Z., Li, J., Zhang, Y.: Mobility-aware seamless service migration and resource allocation in multi-edge IoV systems. IEEE Trans. Mob. Comput. 1, 1\u201317 (2025). https:\/\/doi.org\/10.1109\/TMC.2025.3540407","journal-title":"IEEE Trans. Mob. Comput."},{"key":"5274_CR26","doi-asserted-by":"publisher","unstructured":"Yan, Y., Gao, Y., Chen, Y., Guo, Z., Chen, B., Moscibroda, T.: TR-Spark: transient computing for big data analytics. In: Proceedings of the Seventh ACM Symposium on Cloud Computing, SoCC \u201916, 2016, pp. 484\u2013496. Association for Computing Machinery, New York (2016). https:\/\/doi.org\/10.1145\/2987550.2987576","DOI":"10.1145\/2987550.2987576"},{"key":"5274_CR27","unstructured":"Mehrotra, U.: Spark enhancements for elasticity and resiliency on Amazon EMR (2019). https:\/\/aws.amazon.com\/blogs\/big-data\/spark-enhancements-for-elasticity-and-resiliency-on-amazon-emr\/. Accessed 22 Oct 2024"},{"key":"5274_CR28","unstructured":"Chockalingam, P., Liang, E., Das, T., Stephan, J.Y.: Introducing Databricks Optimized Autoscaling on Apache Spark$$^{\\rm TM}$$ (2018). https:\/\/www.databricks.com\/blog\/2018\/05\/02\/introducing-databricks-optimized-auto-scaling.html. Accessed 22 Oct 2024"},{"key":"5274_CR29","unstructured":"Apache Software Foundation: Hadoop: Fair Scheduler. Apache Software Foundation (nd). https:\/\/hadoop.apache.org\/docs\/stable\/hadoop-yarn\/hadoop-yarn-site\/FairScheduler.html. Accessed 22 Oct 2024"},{"key":"5274_CR30","unstructured":"Awad, A., Awaysheh, F., L\u00f3pez, H.A.: BEST: a unified business process enactment via streams and tables for service computing. arXiv preprint (2025). arXiv:2501.14848"},{"key":"5274_CR31","unstructured":"CERN IT Batch Service Documentation: CERN Batch Service User Guide. https:\/\/batchdocs.web.cern.ch\/. Accessed 22 Oct 2024"},{"issue":"5","key":"5274_CR32","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1145\/2890784","volume":"59","author":"B Burns","year":"2016","unstructured":"Burns, B., Grant, B., Oppenheimer, D., Brewer, E., Wilkes, J.: Borg, Omega, and Kubernetes. Commun. ACM 59(5), 50\u201357 (2016). https:\/\/doi.org\/10.1145\/2890784","journal-title":"Commun. ACM"},{"key":"5274_CR33","doi-asserted-by":"crossref","unstructured":"Kaminaga, H., Awaysheh, F.M., Alawadi, S., Kamm, L.: MPCFL: towards multi-party computation for secure federated learning aggregation. In: Proceedings of the IEEE\/ACM 16th International Conference on Utility and Cloud Computing, 2023, pp. 1\u201310 (2023)","DOI":"10.1145\/3603166.3632144"},{"key":"5274_CR34","doi-asserted-by":"crossref","unstructured":"Nascimento, L., Awaysheh, F.M., Alawadi, S.: Data skew in federated learning: an experimental evaluation on aggregation algorithms. In: 2024 2nd International Conference on Federated Learning Technologies and Applications (FLTA), 2024, pp. 162\u2013170. IEEE (2024)","DOI":"10.1109\/FLTA63145.2024.10840118"},{"key":"5274_CR35","doi-asserted-by":"crossref","unstructured":"Awaysheh, F.M.: From the cloud to the edge towards a distributed and light weight secure big data pipelines for IoT applications. In: Trust, Security and Privacy for Big Data, pp. 50\u201368. CRC Press, Boca Raton (2022)","DOI":"10.1201\/9781003194538-3"},{"key":"5274_CR36","unstructured":"Overview\u2014Spark 3.5.4 Documentation\u2014spark.apache.org. https:\/\/spark.apache.org\/docs\/3.5.4\/index.html"},{"key":"5274_CR37","unstructured":"Tuning\u2014Spark 3.2.1 Documentation\u2014archive.apache.org. https:\/\/archive.apache.org\/dist\/spark\/docs\/3.2.1\/tuning.html#level-of-parallelism"},{"key":"5274_CR38","doi-asserted-by":"publisher","unstructured":"Huang, S., Huang, J., Dai, J., Xie, T., Huang, B.: The HiBench benchmark suite: characterization of the MapReduce-based data analysis. In: 2010 IEEE 26th International Conference on Data Engineering Workshops (ICDEW 2010), 2010, pp. 41\u201351 (2010). https:\/\/doi.org\/10.1109\/ICDEW.2010.5452747","DOI":"10.1109\/ICDEW.2010.5452747"},{"key":"5274_CR39","unstructured":"Google Cloud: General-purpose machine family for Compute Engine. Compute Engine Documentation. https:\/\/cloud.google.com\/compute\/docs\/general-purpose-machines#sharedcore. Accessed 22 Oct 2024"},{"key":"5274_CR40","unstructured":"Agarwal, S.: Google Cloud\u2019s E2 VMs compared to the N2 VMs (2021). https:\/\/www.bigbitbus.com\/2021\/06\/10\/Google-Cloud-E2-N2-VMs\/. Accessed 22 Oct 2024"},{"key":"5274_CR41","doi-asserted-by":"publisher","unstructured":"Yu, B., Cao, H., Shan, T., Wang, H., Tang, X., Chen, W.: Sparker: efficient reduction for more scalable machine learning with Spark. In: Proceedings of the 50th International Conference on Parallel Processing, ICPP \u201921, 2021. Association for Computing Machinery, New York (2021). https:\/\/doi.org\/10.1145\/3472456.3472499","DOI":"10.1145\/3472456.3472499"},{"key":"5274_CR42","doi-asserted-by":"crossref","unstructured":"Awaysheh, F.M., Pena, T.F., Cabaleiro, J.C.: EME: an automated, elastic and efficient prototype for provisioning Hadoop clusters on-demand. In: CLOSER, 2017, pp. 709\u2013714 (2017)","DOI":"10.5220\/0006379607370742"},{"issue":"8","key":"5274_CR43","doi-asserted-by":"publisher","first-page":"1911","DOI":"10.1109\/TPDS.2021.3132422","volume":"33","author":"Z Chen","year":"2022","unstructured":"Chen, Z., Hu, J., Min, G., Luo, C., El-Ghazawi, T.: Adaptive and efficient resource allocation in cloud datacenters using actor\u2013critic deep reinforcement learning. IEEE Trans. Parallel Distrib. Syst. 33(8), 1911\u20131923 (2022). https:\/\/doi.org\/10.1109\/TPDS.2021.3132422","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"key":"5274_CR44","doi-asserted-by":"publisher","DOI":"10.1109\/TMC.2024.3396511","author":"Z Chen","year":"2024","unstructured":"Chen, Z., Xiong, B., Chen, X., Min, G., Li, J.: Joint computation offloading and resource allocation in multi-edge smart communities with personalized federated deep reinforcement learning. IEEE Trans. Mob. Comput. (2024). https:\/\/doi.org\/10.1109\/TMC.2024.3396511","journal-title":"IEEE Trans. Mob. Comput."},{"issue":"12","key":"5274_CR45","doi-asserted-by":"publisher","first-page":"3536","DOI":"10.1109\/JSAC.2024.3459020","volume":"42","author":"Z Chen","year":"2024","unstructured":"Chen, Z., Zhang, J., Min, G., Ning, Z., Li, J.: Traffic-aware lightweight hierarchical offloading toward adaptive slicing-enabled SAGIN. IEEE J. Sel. Areas Commun. 42(12), 3536\u20133550 (2024). https:\/\/doi.org\/10.1109\/JSAC.2024.3459020","journal-title":"IEEE J. Sel. Areas Commun."}],"container-title":["Cluster Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-025-05274-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10586-025-05274-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-025-05274-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,3]],"date-time":"2025-09-03T20:20:42Z","timestamp":1756930842000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10586-025-05274-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,16]]},"references-count":45,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2025,9]]}},"alternative-id":["5274"],"URL":"https:\/\/doi.org\/10.1007\/s10586-025-05274-4","relation":{},"ISSN":["1386-7857","1573-7543"],"issn-type":[{"value":"1386-7857","type":"print"},{"value":"1573-7543","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,6,16]]},"assertion":[{"value":"11 December 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 March 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 March 2025","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 June 2025","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"383"}}