{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:19:34Z","timestamp":1740122374836,"version":"3.37.3"},"reference-count":54,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2022,11,17]],"date-time":"2022-11-17T00:00:00Z","timestamp":1668643200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,11,17]],"date-time":"2022-11-17T00:00:00Z","timestamp":1668643200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100003725","name":"National Research Foundation of Korea","doi-asserted-by":"publisher","award":["NRF-2019R1A2C1005360"],"award-info":[{"award-number":["NRF-2019R1A2C1005360"]}],"id":[{"id":"10.13039\/501100003725","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Cluster Comput"],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1007\/s10586-022-03799-6","type":"journal-article","created":{"date-parts":[[2022,11,17]],"date-time":"2022-11-17T18:13:01Z","timestamp":1668708781000},"page":"2851-2864","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Comprehensive techniques for multi-tenant deep learning framework on a Hadoop YARN cluster"],"prefix":"10.1007","volume":"26","author":[{"given":"Seoungbeom","family":"Heo","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dae-Cheol","family":"Kang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hyeounji","family":"Jang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hyeock-Jin","family":"Lee","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Minkyoung","family":"Cho","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0104-4617","authenticated-orcid":false,"given":"Jik-Soo","family":"Kim","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,11,17]]},"reference":[{"key":"3799_CR1","unstructured":"Abadi, M., Barham, P., Chen, J., Chen, Z., Davis, A., Dean, J., Devin, M., Ghemawat, S., Irving, G., Isard, M., Kudlur, M., Levenberg, J., Monga, R., Moore, S., Murray, D.G., Steiner, B., Tucker, P., Vasudevan, V., Warden, P., Wicke, M., Yu, Y., Zheng, X.: TensorFlow: a system for large-scale machine learning. In: Proceedings of 12th USENIX Symposium on Operating Systems Design and Implementation (OSDI\u201916) (2016)"},{"key":"3799_CR2","unstructured":"TensorFlow: An end-to-end open source machine learning platform. Available at https:\/\/www.tensorflow.org\/"},{"key":"3799_CR3","unstructured":"PyTorch: An open source machine learning framework that accelerates the path from research prototyping to production deployment. Available at https:\/\/pytorch.org\/"},{"key":"3799_CR4","unstructured":"The Apache Hadoop project: open-source software for reliable, scalable, distributed computing. Available at http:\/\/hadoop.apache.org\/"},{"key":"3799_CR5","unstructured":"Dean, J., Corrado, G.S., Monga, R., Chen, K., Devin, M., Le, Q.V., Mao, M.Z., Ranzato, M., Senior, A., Tucker, P., Yang, K., Ng, A.Y.: Large scale distributed deep networks. In: Proceedings of the 25th International Conference on Neural Information Processing Systems (NIPS\u201912) (2012)"},{"key":"3799_CR6","unstructured":"Jiang, Z., Balu, A., Hegde, C., Sarkar, S.: Collaborative deep learning in fixed topology networks. In: Proceedings of the 31st International Conference on Neural Information Processing Systems (NIPS\u201917) (2017)"},{"key":"3799_CR7","unstructured":"Zhang, S., Choromanska, A., LeCun, Y.: Deep learning with elastic averaging SGD. In: Proceedings of the 28th International Conference on Neural Information Processing Systems (NIPS\u201915) (2015)"},{"key":"3799_CR8","doi-asserted-by":"crossref","unstructured":"Shvachko, K., Kuang, H., Radia, S., Chansler, R.: The Hadoop distributed file system. In: Proceedings of the 2010 IEEE 26th Symposium on Mass Storage Systems and Technologies (MSST\u201910) (2010)","DOI":"10.1109\/MSST.2010.5496972"},{"issue":"1","key":"3799_CR9","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1145\/1327452.1327492","volume":"51","author":"J Dean","year":"2008","unstructured":"Dean, J., Ghemawat, S.: MapReduce: simplified data processing on large clusters. Commun. ACM 51(1), 107\u2013113 (2008)","journal-title":"Commun. ACM"},{"key":"3799_CR10","volume-title":"Apache Hadoop YARN: Moving Beyond MapReduce and Batch Processing with Apache Hadoop 2","author":"A Murthy","year":"2014","unstructured":"Murthy, A., Vavilapalli, V., Eadline, D., Niemiec, J., Markham, J.: Apache Hadoop YARN: Moving Beyond MapReduce and Batch Processing with Apache Hadoop 2. Addison-Wesley Data & Analytics, Boston (2014)"},{"key":"3799_CR11","doi-asserted-by":"crossref","unstructured":"Vavilapalli, V.K., Murthy, A.C., Douglas, C., Agarwal, S., Konar, M., Evans, R., Graves, T., Lowe, J., Shah, H., Seth, S., Saha, B., Curino, C., O\u2019Malley, O., Radia, S., Reed, B., Baldeschwieler, E.: Apache Hadoop YARN: yet another resource negotiator. In: Proceedings of the 4th Annual Symposium on Cloud Computing (SoCC\u201913) (2013)","DOI":"10.1145\/2523616.2523633"},{"key":"3799_CR12","doi-asserted-by":"crossref","unstructured":"Huang, J., Zhou, W., Song, R., Liu, F., Wang, S., Liu, J.: Design and performance modeling of A YARN-based GPU resource scheduling system. In: Proceedings of the 2019 IEEE 5th International Conference on Computer and Communications (ICCC) (2019)","DOI":"10.1109\/ICCC47050.2019.9064049"},{"key":"3799_CR13","unstructured":"Sculley, D., Holt, G., Golovin, D., Davydov, E., Phillips, T., Ebner, D., Chaudhary, V., Young, M., Crespo, J.-F., Dennison, D.: Hidden technical debt in machine learning systems. In: Proceedings of the 28th International Conference on Neural Information Processing Systems (NIPS\u201915) (2015)"},{"key":"3799_CR14","doi-asserted-by":"crossref","unstructured":"Kang, D.-C., Heo, S., Jang, H., Lee, H.-J., Cho, M., Kim, J.-S.: MeLoN: Distributed deep learning meets the big data platform. In: Proceedings of the 9th International Workshop on Autonomic Management of High Performance Grid and Cloud Computing (AMGCC\u201921) in Conjunction with ACSOS 2021 (2021)","DOI":"10.1109\/ACSOS-C52956.2021.00028"},{"key":"3799_CR15","unstructured":"Lustre: an open-source, parallel file system that supports many requirements of leadership class HPC simulation environments. Available at https:\/\/www.lustre.org\/"},{"key":"3799_CR16","unstructured":"Schwan, P.: Lustre: building a file system for 1,000-node clusters. In: Proceedings of the Linux Symposium (2003)"},{"key":"3799_CR17","unstructured":"MeLoN: multi-tenant deep learning framework on YARN: Available at https:\/\/github.com\/IDPL-MJU\/MeLoN"},{"key":"3799_CR18","volume-title":"The Enterprise Big Data Lake: Delivering the Promise of Big Data and Data Science","author":"A Gorelik","year":"2019","unstructured":"Gorelik, A.: The Enterprise Big Data Lake: Delivering the Promise of Big Data and Data Science, 1st edn. O\u2019Reilly Media Inc, Sebastopol (2019)","edition":"1"},{"key":"3799_CR19","unstructured":"Apache submarine: a unified AI platform which allows engineers and data scientists to run machine learning and deep learning workload in distributed cluster. Available at https:\/\/submarine.apache.org\/"},{"key":"3799_CR20","unstructured":"Hsu, A., Hu, K., Hung, J., Suresh, A., Zhang, Z.: TonY: an orchestrator for distributed machine learning jobs. In: Proceedings of the 2019 USENIX Conference on Operational Machine Learning (OpML\u201919) (2019)"},{"key":"3799_CR21","unstructured":"Kubernetes: production-grade container orchestration. Available at https:\/\/kubernetes.io\/"},{"key":"3799_CR22","unstructured":"Docker: empowering app development for developers. Available at https:\/\/www.docker.com\/"},{"key":"3799_CR23","unstructured":"Gu, J., Chowdhury, M., Shin, K.G., Zhu, Y., Jeon, M., Qian, J., Liu, H., Guo, C.: Tiresias: a GPU cluster manager for distributed deep learning. In: Proceedings of 16th USENIX Symposium on Networked Systems Design and Implementation (NSDI\u201919) (2019)"},{"key":"3799_CR24","doi-asserted-by":"publisher","unstructured":"Wang, S., Gonzalez, O.J., Zhou, X., Williams, T., Friedman, B.D., Havemann, M., Woo, T.: An efficient and non-intrusive GPU scheduling framework for deep learning training systems. In: SC20: International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201313 (2020). https:\/\/doi.org\/10.1109\/SC41405.2020.00094","DOI":"10.1109\/SC41405.2020.00094"},{"key":"3799_CR25","unstructured":"Mahajan, K., Balasubramanian, A., Singhvi, A., Venkataraman, S., Akella, A., Phanishayee, A., Chawla, S.: Themis: fair and efficient gpu cluster scheduling. In: 17th USENIX Symposium on Networked Systems Design and Implementation (NSDI 20), pp. 289\u2013304. USENIX Association, Santa Clara, CA (2020)"},{"key":"3799_CR26","doi-asserted-by":"publisher","unstructured":"Oh, C., Jung, H., Yi, S., Yoon, I., Yi, Y.: HybridHadoop: CPU-GPU hybrid scheduling in Hadoop. In: The International Conference on High Performance Computing in Asia-Pacific Region. HPC Asia 2021, pp. 40\u201349. Association for Computing Machinery, New York, NY, USA (2021). https:\/\/doi.org\/10.1145\/3432261.3432264","DOI":"10.1145\/3432261.3432264"},{"key":"3799_CR27","unstructured":"Using GPU On YARN. Available at https:\/\/hadoop.apache.org\/docs\/stable\/hadoop-yarn\/hadoop-yarn-site\/UsingGpus.html"},{"key":"3799_CR28","unstructured":"First class GPUs support in Apache Hadoop 3.1, YARN & HDP 3.0. Available at https:\/\/blog.cloudera.com\/gpus-support-in-apache-hadoop-3-1-yarn-hdp-3\/"},{"key":"3799_CR29","unstructured":"Apache Spark: a unified analytics engine for large-scale data processing. Available at https:\/\/spark.apache.org\/"},{"issue":"1","key":"3799_CR30","first-page":"1235","volume":"17","author":"X Meng","year":"2016","unstructured":"Meng, X., Bradley, J., Yavuz, B., Sparks, E., Venkataraman, S., Liu, D., Freeman, J., Tsai, D., Amde, M., Owen, S., Xin, D., Xin, R., Franklin, M.J., Zadeh, R., Zaharia, M., Talwalkar, A.: MLlib: machine learning in apache spark. J. Mach. Learn. Res 17(1), 1235\u20131241 (2016)","journal-title":"J. Mach. Learn. Res"},{"key":"3799_CR31","unstructured":"MLlib: Apache Spark\u2019s scalable machine learning library. Available at https:\/\/spark.apache.org\/mllib\/"},{"key":"3799_CR32","unstructured":"Apache MAHOUT for creating scalable performant machine learning applications. Available at https:\/\/mahout.apache.org\/"},{"issue":"1","key":"3799_CR33","first-page":"1","volume":"21","author":"R Anil","year":"2020","unstructured":"Anil, R., Capan, G., Drost-Fromm, I., Dunning, T., Friedman, E., Grant, T., Quinn, S., Ranjan, P., Schelter, S.: \u00d6zg\u00fcr Yilmazel: Apache Mahout: machine learning on distributed dataflow systems. J. Mach. Learn. Res. 21(1), 1\u20136 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"3799_CR34","doi-asserted-by":"crossref","unstructured":"Dai, J.J., Wang, Y., Qiu, X., Ding, D., Zhang, Y., Wang, Y., Jia, X., Zhang, C.L., Wan, Y., Li, Z., et al.: BigDL: a distributed deep learning framework for big data. In: Proceedings of the ACM Symposium on Cloud Computing. SoCC \u201919 (2019)","DOI":"10.1145\/3357223.3362707"},{"key":"3799_CR35","unstructured":"Sergeev, A., Balso, M.D.: Horovod: fast and easy distributed deep learning in TensorFlow. CoRR abs\/1802.05799 (2018) arXiv:1802.05799"},{"issue":"7","key":"3799_CR36","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1145\/233977.234000","volume":"39","author":"JJ Dongarra","year":"1996","unstructured":"Dongarra, J.J., Otto, S.W., Snir, M., Walker, D.: A message passing standard for MPP and workstations. Commun. ACM 39(7), 84\u201390 (1996)","journal-title":"Commun. ACM"},{"key":"3799_CR37","unstructured":"Jeon, M., Venkataraman, S., Phanishayee, A., Qian, J., Xiao, W., Yang, F.: Analysis of large-scale multi-tenant GPU clusters for DNN training workloads. In: Proceedings of the 2019 USENIX Annual Technical Conference (2019)"},{"key":"3799_CR38","doi-asserted-by":"crossref","unstructured":"Li, M., Andersen, D.G., Park, J.W., Smola, A.J., Ahmed, A., Josifovski, V., Long, J., Shekita, E.J., Su, B.-Y.: Scaling distributed machine learning with the parameter server. In: Proceedings of the 11th USENIX Conference on Operating Systems Design and Implementation (OSDI\u201914) (2014)","DOI":"10.1145\/2640087.2644155"},{"key":"3799_CR39","unstructured":"Bin packing problem (from Wikipedia, the free encyclopedia). Available at https:\/\/en.wikipedia.org\/wiki\/Bin_packing_problem"},{"key":"3799_CR40","unstructured":"THE MNIST DATABASE of handwritten digits. Available at http:\/\/yann.lecun.com\/exdb\/mnist\/"},{"key":"3799_CR41","unstructured":"The CIFAR-10 and CIFAR-100: labeled subsets of the 80 million tiny images dataset. Available at https:\/\/www.cs.toronto.edu\/~kriz\/cifar.html"},{"key":"3799_CR42","doi-asserted-by":"crossref","unstructured":"Kim, S., Hwang, E., Yoo, T.-k., Kim, J.-S., Hwang, S., Choi, Y.-r.: Platform and co-runner affinities for many-task applications in distributed computing platforms. In: Proceedings of the 15th IEEE\/ACM International Symposium on Cluster, Cloud and Grid Computing (CCGrid 2015) (2015)","DOI":"10.1109\/CCGrid.2015.129"},{"key":"3799_CR43","unstructured":"wiki.lustre.org: the main community repository for Lustre information. Available at https:\/\/wiki.lustre.org\/"},{"key":"3799_CR44","unstructured":"Rutman, N.: Map\/reduce on Lustre: hadoop performance in HPC environments. Xyratex White Paper (2011)"},{"key":"3799_CR45","unstructured":"Gallegos, J.M., Tao, Z.: Deploying Hadoop on Lustre storage: lessons learned and best practices. In: The 2015 Lustre User Group (LUG) Conference (2015)"},{"key":"3799_CR46","unstructured":"Skory, S.: Understanding hadoop performance on Lustre. In: The 2015 Lustre User Group (LUG) Conference (2015)"},{"key":"3799_CR47","unstructured":"Hadoop Adapter for Lustre (HAL): Available at https:\/\/github.com\/whamcloud\/lustre-connector-for-hadoop"},{"key":"3799_CR48","unstructured":"The STL-10 dataset: an image recognition dataset for developing unsupervised feature learning, deep learning, self-taught learning algorithms: Available at https:\/\/cs.stanford.edu\/~acoates\/stl10\/"},{"key":"3799_CR49","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"3799_CR50","unstructured":"ImageNet: an image database organized according to the WordNet hierarchy. Available at https:\/\/www.image-net.org\/"},{"key":"3799_CR51","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition (2015)"},{"key":"3799_CR52","unstructured":"Apache Kafka: a high-throughput distributed messaging system: Available at http:\/\/kafka.apache.org\/"},{"key":"3799_CR53","unstructured":"AA Kreps, J., Narkhede, N., Rao, J.: Kafka: a distributed messaging system for log processing. In: Proceedings of the 6th International Workshop on Networking Meets Databases (NetDB\u201911) (2011)"},{"key":"3799_CR54","unstructured":"Apache ActiveMQ: the most popular and powerful open source messaging and integration patterns server. Available at http:\/\/activemq.apache.org\/"}],"container-title":["Cluster Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-022-03799-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10586-022-03799-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-022-03799-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,26]],"date-time":"2023-08-26T20:25:20Z","timestamp":1693081520000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10586-022-03799-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,17]]},"references-count":54,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2023,10]]}},"alternative-id":["3799"],"URL":"https:\/\/doi.org\/10.1007\/s10586-022-03799-6","relation":{},"ISSN":["1386-7857","1573-7543"],"issn-type":[{"type":"print","value":"1386-7857"},{"type":"electronic","value":"1573-7543"}],"subject":[],"published":{"date-parts":[[2022,11,17]]},"assertion":[{"value":"12 March 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 September 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 October 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 November 2022","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Not applicable.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Not applicable.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Not applicable.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}}]}}