{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,17]],"date-time":"2025-11-17T01:30:43Z","timestamp":1763343043100,"version":"3.45.0"},"reference-count":75,"publisher":"Tech Science Press","issue":"3","license":[{"start":{"date-parts":[[2025,5,25]],"date-time":"2025-05-25T00:00:00Z","timestamp":1748131200000},"content-version":"vor","delay-in-days":144,"URL":"https:\/\/doi.org\/10.32604\/TSP-CROSSMARKPOLICY"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["CMC"],"published-print":{"date-parts":[[2025]]},"DOI":"10.32604\/cmc.2025.063047","type":"journal-article","created":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T05:08:41Z","timestamp":1743484121000},"page":"3843-3875","update-policy":"https:\/\/doi.org\/10.32604\/tsp-crossmarkpolicy","source":"Crossref","is-referenced-by-count":0,"title":["A Survey of Spark Scheduling Strategy Optimization Techniques and Development Trends"],"prefix":"10.32604","volume":"83","author":[{"given":"Chuan","family":"Li","sequence":"first","affiliation":[]},{"given":"Xuanlin","family":"Wen","sequence":"additional","affiliation":[]}],"member":"17807","published-online":{"date-parts":[[2025]]},"reference":[{"key":"ref1","first-page":"1770","article-title":"Comparative study on MapReduce and spark for big data analytics","volume":"29","author":"Wu","year":"2018","journal-title":"J Softw"},{"key":"ref2","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1145\/2934664","article-title":"Apache spark: a unified engine for big data processing","volume":"59","author":"Zaharia","year":"2016","journal-title":"Commun ACM"},{"key":"ref3","series-title":"9th USENIX Symposium on Networked Systems Design and Implementation (NSDI 12)","first-page":"15","article-title":"Resilient distributed datasets: a fault-tolerant abstraction for in-memory cluster computing","author":"Zaharia","year":"2012 Apr 25\u201327"},{"key":"ref4","series-title":"2nd USENIX Workshop on Hot Topics in Cloud Computing (HotCloud 10)","article-title":"Spark: cluster computing with working sets","author":"Zaharia","year":"2010 Jun 22"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1186\/s40537-019-0240-1","article-title":"Leveraging resource management for efficient performance of Apache Spark","volume":"6","author":"Aziz","year":"2019","journal-title":"J Big Data"},{"key":"ref6","first-page":"71","article-title":"A survey on spark ecosystem: big data processing infrastructure, machine learning, and applications","volume":"34","author":"Tang","year":"2022","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"ref7","first-page":"7","article-title":"Survey on performance optimization technologies for spark","volume":"45","author":"Liao","year":"2018","journal-title":"Comput Sci"},{"key":"ref8","first-page":"137","article-title":"Survey on storage and optimization techniques of HDFS","volume":"31","author":"Jin","year":"2020","journal-title":"J Softw"},{"key":"ref9","first-page":"463","article-title":"Survey on JVM optimization for big data processing frameworks","volume":"34","author":"Wang","year":"2021","journal-title":"J Softw"},{"key":"ref10","doi-asserted-by":"crossref","DOI":"10.1145\/2886107","author":"Zaharia","year":"2016","journal-title":"An architecture for fast and general data processing on large clusters"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"131885","DOI":"10.1109\/ACCESS.2020.3007499","article-title":"Parallel machine learning algorithm using fine-grained-mode spark on a mesos big data cloud computing software framework for mobile robotic intelligent fault recognition","volume":"8","author":"Xian","year":"2020","journal-title":"IEEE Access"},{"key":"ref12","series-title":"2020 IEEE International Parallel and Distributed Processing Symposium (IPDPS)","first-page":"378","article-title":"DAG-aware joint task scheduling and cache management in spark clusters","author":"Xu","year":"2020 May 18\u201322"},{"key":"ref13","series-title":"2018 18th IEEE\/ACM International Symposium on Cluster, Cloud and Grid Computing (CCGRID)","first-page":"645","article-title":"A hard real-time scheduler for spark on YARN","author":"Wang","year":"2018 May 1\u20134"},{"key":"ref14","series-title":"2021 20th International Symposium on Parallel and Distributed Computing (ISPDC)","first-page":"97","article-title":"Efficient real-time earliest deadline first based scheduling for Apache spark","author":"Neciu","year":"2021 Jul 28\u201330"},{"key":"ref15","series-title":"2018 IEEE 11th International Conference on Cloud Computing (CLOUD)","first-page":"434","article-title":"Exploring the fairness and resource distribution in an Apache mesos environment","author":"Saha","year":"2018 Jul 2\u20137"},{"key":"ref16","first-page":"230","article-title":"Multicore architecture speedup computation based on Amdahl\u2019s law and Rent\u2019s rule","volume":"40","author":"Li","year":"2012","journal-title":"Dianzi Xuebao Acta Electron Sin"},{"key":"ref17","first-page":"8598305","article-title":"PAS: performance-aware job scheduling for big data processing systems","volume":"2022","author":"Li","year":"2022","journal-title":"Secur Commun Netw"},{"key":"ref18","first-page":"567","article-title":"Parallelism deduction algorithm for spark","volume":"48","author":"Bian","year":"2019","journal-title":"J Univ Electron Sci Technol China"},{"key":"ref19","doi-asserted-by":"crossref","first-page":"102776","DOI":"10.1016\/j.eti.2022.102776","article-title":"Spatiotemporal data partitioning for distributed random forest algorithm: air quality prediction using imbalanced big spatiotemporal data on spark distributed framework","volume":"27","author":"Asgari","year":"2022","journal-title":"Environ Technol Innov"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"2406","DOI":"10.1109\/TPDS.2020.2992073","article-title":"An optimal locality-aware task scheduling algorithm based on bipartite graph modelling for spark applications","volume":"31","author":"Fu","year":"2020","journal-title":"IEEE Trans Parallel Distrib Syst"},{"key":"ref21","first-page":"176","article-title":"Adaptive scheduling strategy based on deadline under cloud platform","volume":"43","author":"Wu","year":"2023","journal-title":"J Comput Appl"},{"key":"ref22","first-page":"173","article-title":"Design of distributed computing framework for foreign exchange market monitoring","volume":"40","author":"Cheng","year":"2020","journal-title":"J Comput Appl"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"3588","DOI":"10.1109\/TCYB.2020.3026673","article-title":"Scheduling-guided automatic processing of massive hyperspectral image classification on cloud computing architectures","volume":"51","author":"Wu","year":"2021","journal-title":"IEEE Trans Cybern"},{"key":"ref24","doi-asserted-by":"crossref","first-page":"587","DOI":"10.1109\/JSTARS.2020.3036896","article-title":"Multiobjective task scheduling for energy-efficient cloud implementation of hyperspectral image classification","volume":"14","author":"Sun","year":"2020","journal-title":"IEEE J Sel Top Appl Earth Obs Remote Sens"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"1376","DOI":"10.1109\/TC.2019.2906193","article-title":"Semi-clairvoyant scheduling in data analytics systems","volume":"68","author":"Zhang","year":"2019","journal-title":"IEEE Trans Comput"},{"key":"ref26","doi-asserted-by":"crossref","first-page":"1668","DOI":"10.1109\/TSE.2019.2931537","article-title":"Fine-grained dynamic resource allocation for big-data applications","volume":"47","author":"Baresi","year":"2021","journal-title":"IEEE Trans Softw Eng"},{"key":"ref27","doi-asserted-by":"crossref","first-page":"1874","DOI":"10.3390\/electronics10161874","article-title":"Performance improvement of DAG-aware task scheduling algorithms with efficient cache management in spark","volume":"10","author":"Zhao","year":"2021","journal-title":"Electronics"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"2793","DOI":"10.1109\/ACCESS.2020.3040719","article-title":"Scheduling spark tasks with data skew and deadline constraints","volume":"9","author":"Gu","year":"2020","journal-title":"IEEE Access"},{"key":"ref29","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1016\/j.future.2024.02.009","article-title":"PAC: a monitoring framework for performance analysis of compression algorithms in Spark","volume":"157","author":"Zhu","year":"2024","journal-title":"Future Gener Comput Syst"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"116","DOI":"10.1016\/j.future.2024.03.005","article-title":"DAG-aware harmonizing job scheduling and data caching for disaggregated analytics frameworks","volume":"156","author":"Tong","year":"2024","journal-title":"Future Gener Comput Syst"},{"key":"ref31","first-page":"203","article-title":"Task scheduling optimization in Spark environment with unbalanced resources","volume":"42","author":"Hu","year":"2020","journal-title":"Comput Eng Sci"},{"key":"ref32","doi-asserted-by":"crossref","first-page":"639","DOI":"10.1109\/TCC.2021.3108043","article-title":"Dynamic resource provisioning for iterative workloads on Apache spark","volume":"11","author":"Cheng","year":"2023","journal-title":"IEEE Trans Cloud Comput"},{"key":"ref33","first-page":"299","article-title":"Dynamic data partition based on node load","volume":"30","author":"Meng","year":"2021","journal-title":"Comput Syst Appl"},{"key":"ref34","first-page":"102","article-title":"Node priority optimization in distributed heterogeneous clusters","volume":"44","author":"Hu","year":"2022","journal-title":"J Natl Univ Def Technol"},{"key":"ref35","doi-asserted-by":"crossref","first-page":"2971","DOI":"10.3390\/s21092971","article-title":"On-the-fly fusion of remotely-sensed big data using an elastic computing paradigm with a containerized spark engine on Kubernetes","volume":"21","author":"Huang","year":"2021","journal-title":"Sensors"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"110","DOI":"10.1186\/s40537-020-00388-5","article-title":"A comprehensive performance analysis of Apache Hadoop and Apache Spark for large scale data sets using HiBench","volume":"7","author":"Ahmed","year":"2020","journal-title":"J Big Data"},{"key":"ref37","first-page":"327","article-title":"Heterogeneous cluster resource allocation algorithm considering application and node characteristics","volume":"58","author":"Hu","year":"2022","journal-title":"J Comput Eng Appl"},{"key":"ref38","doi-asserted-by":"crossref","first-page":"101801","DOI":"10.1016\/j.sysarc.2020.101801","article-title":"A holistic cross-layer optimization approach for mitigating stragglers in in-memory data processing","volume":"111","author":"Zhao","year":"2020","journal-title":"J Syst Archit"},{"key":"ref39","first-page":"620","article-title":"A data skew correction scheduling strategy of heterogeneous Spark cluster","volume":"44","author":"Bian","year":"2022","journal-title":"Comput Eng Sci"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"110515","DOI":"10.1016\/j.jss.2019.110515","article-title":"Cost-efficient dynamic scheduling of big data applications in Apache spark on cloud","volume":"162","author":"Islam","year":"2020","journal-title":"J Syst Softw"},{"key":"ref41","first-page":"180","article-title":"Distributed parallel task scheduling on spark-gpu framework for oceanographic geospatial data processing","volume":"48","author":"Jing","year":"2018","journal-title":"Period Ocean Univ China"},{"key":"ref42","doi-asserted-by":"crossref","first-page":"107050","DOI":"10.1016\/j.knosys.2021.107050","article-title":"Adaptive priority-based data placement and multi-task scheduling in geo-distributed cloud systems","volume":"224","author":"Li","year":"2021","journal-title":"Knowl Based Syst"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"73","DOI":"10.1007\/s10619-023-07436-y","article-title":"SimCost: cost-effective resource provision prediction and recommendation for spark workloads","volume":"42","author":"Chen","year":"2024","journal-title":"Distrib Parallel Databases"},{"key":"ref44","doi-asserted-by":"crossref","first-page":"876","DOI":"10.1109\/TCC.2024.3406041","article-title":"Improving data locality of tasks by executor allocation in spark computing environment","volume":"12","author":"Fu","year":"2024","journal-title":"IEEE Trans Cloud Comput"},{"key":"ref45","doi-asserted-by":"crossref","first-page":"57372","DOI":"10.1109\/ACCESS.2021.3072609","article-title":"Distributed tree-based machine learning for short-term load forecasting with Apache spark","volume":"9","author":"Zainab","year":"2021","journal-title":"IEEE Access"},{"key":"ref46","doi-asserted-by":"crossref","first-page":"107092","DOI":"10.1016\/j.compeleceng.2021.107092","volume":"92","author":"Du","year":"2021","journal-title":"Comput Electr Eng"},{"key":"ref47","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1016\/j.jpdc.2020.03.010","article-title":"Dynamic memory-aware scheduling in spark computing environment","volume":"141","author":"Tang","year":"2020","journal-title":"J Parallel Distrib Comput"},{"key":"ref48","doi-asserted-by":"crossref","first-page":"e5823","DOI":"10.1002\/cpe.5823","article-title":"Mary, Hugo, and Hugo*: learning to schedule distributed data-parallel processing jobs on shared clusters","volume":"33","author":"Thamsen","year":"2021","journal-title":"Concurr Comput Pract Exp"},{"key":"ref49","first-page":"228","article-title":"Workflows scheduling powered by execution time prediction model","volume":"46","author":"Hu","year":"2024","journal-title":"J Natl Univ Def Technol"},{"key":"ref50","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1007\/s10723-024-09756-4","article-title":"Adaptive scheduling framework of streaming applications based on resource demand prediction with hybrid algorithms","volume":"22","author":"Li","year":"2024","journal-title":"J Grid Comput"},{"key":"ref51","doi-asserted-by":"crossref","first-page":"57822","DOI":"10.1109\/ACCESS.2020.2982320","article-title":"Hawkeye: adaptive straggler identification on heterogeneous spark cluster with reinforcement learning","volume":"8","author":"Du","year":"2020","journal-title":"IEEE Access"},{"key":"ref52","doi-asserted-by":"crossref","first-page":"29","DOI":"10.31449\/inf.v48i20.6779","article-title":"Distributed intelligent optimization of e-commerce user purchase data mining using spark framework","volume":"48","author":"Wu","year":"2024","journal-title":"Informatica"},{"key":"ref53","doi-asserted-by":"crossref","first-page":"e6316","DOI":"10.1002\/cpe.6316","article-title":"Big data analytics for identifying electricity theft using machine learning approaches in microgrids for smart communities","volume":"33","author":"Arif","year":"2021","journal-title":"Concurr Comput Pract Exp"},{"key":"ref54","doi-asserted-by":"crossref","first-page":"100373","DOI":"10.1016\/j.array.2024.100373","article-title":"Mining area skyline objects from map-based big data using Apache Spark framework","volume":"25","author":"Li","year":"2025","journal-title":"Array"},{"key":"ref55","doi-asserted-by":"crossref","first-page":"11078","DOI":"10.1007\/s11227-023-05100-x","article-title":"A new Apache Spark-based framework for big data streaming forecasting in IoT networks","volume":"79","author":"Fern\u00e1ndez-G\u00f3mez","year":"2023","journal-title":"J Supercomput"},{"key":"ref56","first-page":"608","article-title":"Apache spark in healthcare: advancing data-driven innovations and better patient care","volume":"14","author":"Shrotriya","year":"2023","journal-title":"Int J Adv Comput Sci Appl"},{"key":"ref57","doi-asserted-by":"crossref","first-page":"97","DOI":"10.1186\/s40537-019-0260-x","article-title":"An adaptive and real-time based architecture for financial data integration","volume":"6","author":"Fikri","year":"2019","journal-title":"J Big Data"},{"key":"ref58","first-page":"1790","article-title":"A survey on software-hardware acceleration for fully homomorphic encryption","volume":"46","author":"Bian","year":"2024","journal-title":"J Electron Inf Technol"},{"key":"ref59","doi-asserted-by":"crossref","first-page":"237","DOI":"10.35833\/MPCE.2020.000472","article-title":"Electric load clustering in smart grid: methodologies, applications, and future trends","volume":"9","author":"Si","year":"2021","journal-title":"J Mod Power Syst Clean Energy"},{"key":"ref60","doi-asserted-by":"crossref","first-page":"1875","DOI":"10.1080\/00207543.2020.1793011","article-title":"Role of big data analytics in supply chain management: current trends and future perspectives.","volume":"59","author":"Maheshwari","year":"2021","journal-title":"Int J Prod Res"},{"key":"ref61","doi-asserted-by":"crossref","first-page":"107","DOI":"10.1186\/s40537-021-00499-7","article-title":"A parallelization model for performance characterization of Spark Big Data jobs on Hadoop clusters","volume":"8","author":"Ahmed","year":"2021","journal-title":"J Big Data"},{"key":"ref62","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1186\/s40537-022-00623-1","article-title":"Runtime prediction of big data jobs: performance comparison of machine learning algorithms and analytical models","volume":"9","author":"Ahmed","year":"2022","journal-title":"J Big Data"},{"key":"ref63","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1186\/s40537-016-0051-6","article-title":"Analyzing performance of Apache Tez and MapReduce with hadoop multinode cluster on Amazon cloud","volume":"3","author":"Singh","year":"2016","journal-title":"J Big Data"},{"key":"ref64","doi-asserted-by":"crossref","first-page":"637","DOI":"10.1109\/JIOT.2016.2579198","article-title":"Edge computing: vision and challenges","volume":"3","author":"Shi","year":"2016","journal-title":"IEEE Internet Things J"},{"key":"ref65","first-page":"2406","article-title":"Survey on the methods of computation offloading in mobile edge computing","volume":"44","author":"Zhang","year":"2021","journal-title":"J Comput Sci"},{"key":"ref66","doi-asserted-by":"crossref","first-page":"1175","DOI":"10.1360\/N112016-00147","article-title":"Emerging high-performance computing systems and technology","volume":"46","author":"Liao","year":"2016","journal-title":"Sci Sin-Inf"},{"key":"ref67","first-page":"2152","article-title":"State-of-the-art survey of real-time multicore system","volume":"29","author":"Chen","year":"2018","journal-title":"J Softw"},{"key":"ref68","doi-asserted-by":"crossref","first-page":"5547","DOI":"10.1007\/s10586-023-04257-7","article-title":"Scalability of blockchain: a comprehensive review and future research direction","volume":"27","author":"Rao","year":"2024","journal-title":"Clust Comput"},{"key":"ref69","doi-asserted-by":"crossref","first-page":"3801","DOI":"10.3390\/electronics12183801","article-title":"Byzantine fault-tolerant consensus algorithms: a survey","volume":"12","author":"Zhong","year":"2023","journal-title":"Electronics"},{"key":"ref70","first-page":"550","article-title":"Design and implementation of trusted fusion and sharing model of agricultural data based on Blockchain","volume":"50","author":"Rao","year":"2023","journal-title":"J Anhui Agric Univ"},{"key":"ref71","doi-asserted-by":"crossref","first-page":"108957","DOI":"10.1016\/j.comnet.2022.108957","article-title":"Energy-aware task scheduling and offloading using deep reinforcement learning in SDN-enabled IoT network","volume":"210","author":"Sellami","year":"2022","journal-title":"Comput Netw"},{"key":"ref72","first-page":"1","article-title":"Task scheduling and load balancing in SDN-based cloud computing: a review of relevant research","volume":"2024","author":"Mahdizadeh","year":"2024","journal-title":"J Eng Res"},{"key":"ref73","doi-asserted-by":"crossref","first-page":"109","DOI":"10.1016\/B978-0-32-395365-8.00014-2","author":"Salehnia","year":"2024","journal-title":"Handbook of whale optimization algorithm"},{"key":"ref74","series-title":"Proceedings of the Fifteenth European Conference on Computer Systems","first-page":"1","article-title":"Borg: the next generation","author":"Tirmazi","year":"2020 Apr 27\u201330"},{"key":"ref75","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3539606","article-title":"Kubernetes scheduling: taxonomy, ongoing issues and challenges","volume":"55","author":"Carri\u00f3n","year":"2023","journal-title":"ACM Comput Surv"}],"container-title":["Computers, Materials & Continua"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/cdn.techscience.cn\/files\/cmc\/2025\/TSP_CMC-83-3\/TSP_CMC_63047\/TSP_CMC_63047.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,17]],"date-time":"2025-11-17T01:27:50Z","timestamp":1763342870000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.techscience.com\/cmc\/v83n3\/61030"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":75,"journal-issue":{"issue":"3","published-online":{"date-parts":[[2025]]},"published-print":{"date-parts":[[2025]]}},"URL":"https:\/\/doi.org\/10.32604\/cmc.2025.063047","relation":{},"ISSN":["1546-2226"],"issn-type":[{"type":"electronic","value":"1546-2226"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"2025-01-03","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-03-06","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-05-19","order":2,"name":"published","label":"Published Online","group":{"name":"publication_history","label":"Publication History"}}]}}