{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,12]],"date-time":"2026-09-12T01:39:27Z","timestamp":1789177167158,"version":"build-2803163510"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,22]]},"DOI":"10.1145\/3722212.3724451","type":"proceedings-article","created":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T09:00:26Z","timestamp":1750150826000},"page":"743-756","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Rockhopper: A Robust Optimizer for Spark Configuration Tuning in Production Environment"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-6857-7505","authenticated-orcid":false,"given":"Yiwen","family":"Zhu","sequence":"first","affiliation":[{"name":"Microsoft, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4736-2837","authenticated-orcid":false,"given":"Rathijit","family":"Sen","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5108-6743","authenticated-orcid":false,"given":"Brian","family":"Kroth","sequence":"additional","affiliation":[{"name":"Microsoft, Madison, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4774-0965","authenticated-orcid":false,"given":"Sergiy","family":"Matusevych","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2349-9428","authenticated-orcid":false,"given":"Andreas","family":"Mueller","sequence":"additional","affiliation":[{"name":"Microsoft, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-8658-271X","authenticated-orcid":false,"given":"Tengfei","family":"Huang","sequence":"additional","affiliation":[{"name":"Microsoft, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5313-9690","authenticated-orcid":false,"given":"Rahul","family":"Challapalli","sequence":"additional","affiliation":[{"name":"Microsoft, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-0454-7490","authenticated-orcid":false,"given":"Weihan","family":"Tang","sequence":"additional","affiliation":[{"name":"Microsoft, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-0939-9669","authenticated-orcid":false,"given":"Xin","family":"He","sequence":"additional","affiliation":[{"name":"Microsoft, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-4913-1466","authenticated-orcid":false,"given":"Mo","family":"Liu","sequence":"additional","affiliation":[{"name":"Microsoft, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8308-7129","authenticated-orcid":false,"given":"Estera","family":"Kot","sequence":"additional","affiliation":[{"name":"Clouds on Mars, Warsaw, Poland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3276-6030","authenticated-orcid":false,"given":"Sule","family":"Kahraman","sequence":"additional","affiliation":[{"name":"Microsoft, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5289-3320","authenticated-orcid":false,"given":"Arshdeep","family":"Sekhon","sequence":"additional","affiliation":[{"name":"Microsoft, Fort Mill, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-2616-7848","authenticated-orcid":false,"given":"Dario","family":"Bernal","sequence":"additional","affiliation":[{"name":"Microsoft, Cambridge, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4748-8589","authenticated-orcid":false,"given":"Aditya","family":"Lakra","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-2512-2142","authenticated-orcid":false,"given":"Shaily","family":"Fozdar","sequence":"additional","affiliation":[{"name":"Dynamo AI, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6763-4035","authenticated-orcid":false,"given":"Dhruv","family":"Relwani","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-6045-7037","authenticated-orcid":false,"given":"Rui","family":"Fang","sequence":"additional","affiliation":[{"name":"Microsoft, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-6121-5312","authenticated-orcid":false,"given":"Long","family":"Tian","sequence":"additional","affiliation":[{"name":"Microsoft, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-9925-7484","authenticated-orcid":false,"given":"Karuna Sagar","family":"Krishna","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7939-6692","authenticated-orcid":false,"given":"Ashit","family":"Gosalia","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3712-7358","authenticated-orcid":false,"given":"Carlo","family":"Curino","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-8534-0889","authenticated-orcid":false,"given":"Subru","family":"Krishnan","sequence":"additional","affiliation":[{"name":"Microsoft, Barcelona, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,6,22]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Anonymous. 2024. AutoDebugger: Efficient Root Cause Analysis for Anomaly Jobs. (2024). Working paper."},{"key":"e_1_3_2_1_2_1","unstructured":"AWS. 2022. Amazon.com Inc. https:\/\/aws.amazon.com\/emr\/features\/spark\/ Retrieved July 2 2022 from"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2015.2449299"},{"key":"e_1_3_2_1_4_1","volume-title":"Pattern recognition and machine learning","author":"Bishop Christopher M","unstructured":"Christopher M Bishop and Nasser M Nasrabadi. 2006. Pattern recognition and machine learning. Vol. 4. Springer."},{"key":"e_1_3_2_1_5_1","volume-title":"https:\/\/scikit-optimize.github.io\/stable\/index.html Retrieved","author":"Contributors Scikit-Optimize","year":"2023","unstructured":"Scikit-Optimize Contributors. 2023. Scikit-Optimize. https:\/\/scikit-optimize.github.io\/stable\/index.html Retrieved July 2, 2023 from"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3399579.3399927"},{"key":"e_1_3_2_1_7_1","volume-title":"Tuneful: An Online Significance-Aware Configuration Tuner for Big Data Analytics. CoRR","author":"Fekry Ayat","year":"2020","unstructured":"Ayat Fekry, Lucian Carata, Thomas F. J.-M. Pasquier, Andrew Rice, and Andy Hopper. 2020. Tuneful: An Online Significance-Aware Configuration Tuner for Big Data Analytics. CoRR, Vol. abs\/2001.08002 (2020). showeprint[arXiv]2001.08002 https:\/\/arxiv.org\/abs\/2001.08002"},{"key":"e_1_3_2_1_8_1","unstructured":"Johannes Freischuetz Konstantinos Kanellis Brian Kroth and Shivaram Venkataraman. [n.d.]. Performance Roulette: How Cloud Weather Affects ML-Based System Optimization."},{"key":"e_1_3_2_1_9_1","volume-title":"https:\/\/cloud.google.com\/dataproc-serverless\/docs Retrieved","author":"Spark Serverless","year":"2022","unstructured":"Google. 2022a. Serverless Spark. https:\/\/cloud.google.com\/dataproc-serverless\/docs Retrieved July 2, 2022 from"},{"key":"e_1_3_2_1_10_1","volume-title":"Spark through Vertex AI. https:\/\/cloud.google.com\/vertex-ai-workbench Retrieved","year":"2022","unstructured":"Google. 2022b. Spark through Vertex AI. https:\/\/cloud.google.com\/vertex-ai-workbench Retrieved July 2, 2022 from"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICC.2018.8422658"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.3390\/s22155930"},{"key":"e_1_3_2_1_13_1","first-page":"2011","article-title":"Sequential model-based optimization for general algorithm configuration. In Learning and Intelligent Optimization: 5th International Conference","volume":"21","author":"Hutter Frank","year":"2011","unstructured":"Frank Hutter, Holger H Hoos, and Kevin Leyton-Brown. 2011. Sequential model-based optimization for general algorithm configuration. In Learning and Intelligent Optimization: 5th International Conference, LION 5, Rome, Italy, January 17--21, 2011. Selected Papers 5. Springer, 507--523.","journal-title":"LION 5, Rome, Italy"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357223.3362726"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.14778\/3551793.3551844"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.14778\/3685800.3685852"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.14778\/3611540.3611548"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE53745.2022.00195"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.14778\/3681954.3682021"},{"key":"e_1_3_2_1_20_1","volume-title":"https:\/\/docs.microsoft.com\/en-us\/azure\/hdinsight\/spark\/apache-spark-overview Retrieved","author":"Insight Azure","year":"2022","unstructured":"Microsoft. 2022a. Azure HDInsight. https:\/\/docs.microsoft.com\/en-us\/azure\/hdinsight\/spark\/apache-spark-overview Retrieved July 2, 2022 from"},{"key":"e_1_3_2_1_21_1","volume-title":"https:\/\/docs.microsoft.com\/en-us\/azure\/synapse-analytics\/spark\/apache-spark-overview Retrieved","author":"Synapse Azure","year":"2022","unstructured":"Microsoft. 2022b. Azure Synapse. https:\/\/docs.microsoft.com\/en-us\/azure\/synapse-analytics\/spark\/apache-spark-overview Retrieved July 2, 2022 from"},{"key":"e_1_3_2_1_22_1","volume-title":"https:\/\/learn.microsoft.com\/en-us\/nimbusml\/overview Retrieved","author":"ML.","year":"2023","unstructured":"Microsoft. 2023. NimbusML. https:\/\/learn.microsoft.com\/en-us\/nimbusml\/overview Retrieved April 5, 2023 from"},{"key":"e_1_3_2_1_23_1","volume-title":"Configure Autotune for Fabric Spark. https:\/\/learn.microsoft.com\/en-us\/fabric\/data-engineering\/autotune?tabs=sparksql Retrieved","year":"2024","unstructured":"Microsoft. 2024. Configure Autotune for Fabric Spark. https:\/\/learn.microsoft.com\/en-us\/fabric\/data-engineering\/autotune?tabs=sparksql Retrieved June 27, 2024 from"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/CLOUD.2018.00059"},{"key":"e_1_3_2_1_25_1","volume-title":"Open Neural Network Exchange (ONNX). https:\/\/onnx.ai\/ Retrieved","author":"ONNX.AI.","year":"2023","unstructured":"ONNX.AI. 2023. Open Neural Network Exchange (ONNX). https:\/\/onnx.ai\/ Retrieved April 5, 2023 from"},{"key":"e_1_3_2_1_26_1","volume-title":"https:\/\/en.wikipedia.org\/wiki\/Hill_climbing Retrieved","author":"Pedia Wiki","year":"2023","unstructured":"Wiki Pedia. 2023. Hill Climbing. https:\/\/en.wikipedia.org\/wiki\/Hill_climbing Retrieved Oct 18, 2023 from"},{"key":"e_1_3_2_1_27_1","volume-title":"Mach. Learn. Res.","volume":"12","author":"Pedregosa Fabian","year":"2011","unstructured":"Fabian Pedregosa, Ga\u00ebl Varoquaux, Alexandre Gramfort, Vincent Michel, Bertrand Thirion, Olivier Grisel, Mathieu Blondel, Peter Prettenhofer, Ron Weiss, Vincent Dubourg, Jake Vanderplas, Alexandre Passos, David Cournapeau, Matthieu Brucher, Matthieu Perrot, and \u00c9douard Duchesnay. 2011. Scikit-learn: Machine Learning in Python. J. Mach. Learn. Res., Vol. 12 (Nov. 2011), 2825--2830. http:\/\/dl.acm.org\/citation.cfm?id=1953048.2078195"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/TNSM.2020.3034824"},{"key":"e_1_3_2_1_29_1","volume-title":"On the momentum term in gradient descent learning algorithms. Neural networks","author":"Qian Ning","year":"1999","unstructured":"Ning Qian. 1999. On the momentum term in gradient descent learning algorithms. Neural networks, Vol. 12, 1 (1999), 145--151."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476311.3476388"},{"key":"e_1_3_2_1_31_1","volume-title":"Support Vector Machine. https:\/\/scikit-learn.org\/stable\/modules\/generated\/sklearn.svm.SVR.html Retrieved","year":"2023","unstructured":"Scikit-learn. 2023. Support Vector Machine. https:\/\/scikit-learn.org\/stable\/modules\/generated\/sklearn.svm.SVR.html Retrieved Nov 22, 2023 from"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.48786\/EDBT.2023.10"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476311.3476362"},{"key":"e_1_3_2_1_34_1","volume-title":"Rover: An online Spark SQL tuning service via generalized transfer learning. arXiv preprint arXiv:2302.04046","author":"Shen Yu","year":"2023","unstructured":"Yu Shen, Xinyuyang Ren, Yupeng Lu, Huaijun Jiang, Huanyong Xu, Di Peng, Yang Li, Wentao Zhang, and Bin Cui. 2023. Rover: An online Spark SQL tuning service via generalized transfer learning. arXiv preprint arXiv:2302.04046 (2023)."},{"key":"e_1_3_2_1_35_1","volume-title":"21st USENIX Symposium on Networked Systems Design and Implementation (NSDI 24)","author":"Somashekar Gagan","year":"2024","unstructured":"Gagan Somashekar, Karan Tandon, Anush Kini, Chieh-Chun Chang, Petr Husak, Ranjita Bhagwan, Mayukh Das, Anshul Gandhi, and Nagarajan Natarajan. 2024. {OPPerTune}:{Post-Deployment} Configuration Tuning of Services Made Easy. In 21st USENIX Symposium on Networked Systems Design and Implementation (NSDI 24). 1101--1120."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE51399.2021.00041"},{"key":"e_1_3_2_1_37_1","volume-title":"International conference on machine learning. PMLR, 1139--1147","author":"Sutskever Ilya","year":"2013","unstructured":"Ilya Sutskever, James Martens, George Dahl, and Geoffrey Hinton. 2013. On the importance of initialization and momentum in deep learning. In International conference on machine learning. PMLR, 1139--1147."},{"key":"e_1_3_2_1_38_1","volume-title":"Retrieved","author":"Union European","year":"2022","unstructured":"European Union. 2022. General Data Protection Regulation. https:\/\/eur-lex.europa.eu\/legal-content\/EN\/ALL\/?uri=celex. Retrieved Feb 23, 2022 from"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3064029"},{"key":"e_1_3_2_1_40_1","unstructured":"Chi Wang Qingyun Wu Silu Huang and Amin Saied. 2021a. Economical Hyperparameter Optimization With Blended Search Strategy. In ICLR."},{"key":"e_1_3_2_1_41_1","volume-title":"FLAML: A Fast and Lightweight AutoML Library. In MLSys.","author":"Wang Chi","year":"2021","unstructured":"Chi Wang, Qingyun Wu, Markus Weimer, and Erkang Zhu. 2021b. FLAML: A Fast and Lightweight AutoML Library. In MLSys."},{"key":"e_1_3_2_1_42_1","volume-title":"A novel method for tuning configuration parameters of spark based on machine learning. In 2016 IEEE 18th International Conference on High Performance Computing and Communications","author":"Wang Guolu","unstructured":"Guolu Wang, Jungang Xu, and Ben He. 2016. A novel method for tuning configuration parameters of spark based on machine learning. In 2016 IEEE 18th International Conference on High Performance Computing and Communications; IEEE 14th International Conference on Smart City; IEEE 2nd International Conference on Data Science and Systems (HPCC\/SmartCity\/DSS). IEEE, 586--593."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.14778\/3641204.3641205"},{"key":"e_1_3_2_1_44_1","unstructured":"Qingyun Wu Chi Wang and Silu Huang. 2021. Frugal Optimization for Cost-related Hyperparameters. In AAAI."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.14778\/3685800.3685804"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3514221.3526157"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173162.3173187"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3236222"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299869.3300085"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.14778\/3681954.3682007"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.14778\/3538598.3538604"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3514221.3526176"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476249.3476298"},{"key":"e_1_3_2_1_54_1","volume-title":"Towards Building Autonomous Data Services on Azure. In Companion of the 2023 International Conference on Management of Data. 217--224","author":"Zhu Yiwen","year":"2023","unstructured":"Yiwen Zhu, Yuanyuan Tian, Joyce Cahoon, Subru Krishnan, Ankita Agarwal, Rana Alotaibi, Jes\u00fas Camacho-Rodriguez, Bibin Chundatt, Andrew Chung, Niharika Dutta, et al. 2023. Towards Building Autonomous Data Services on Azure. In Companion of the 2023 International Conference on Management of Data. 217--224."}],"event":{"name":"SIGMOD\/PODS '25: International Conference on Management of Data","location":"Berlin Germany","acronym":"SIGMOD\/PODS '25","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Companion of the 2025 International Conference on Management of Data"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3722212.3724451","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,10]],"date-time":"2025-09-10T22:41:42Z","timestamp":1757544102000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3722212.3724451"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,22]]},"references-count":54,"alternative-id":["10.1145\/3722212.3724451","10.1145\/3722212"],"URL":"https:\/\/doi.org\/10.1145\/3722212.3724451","relation":{},"subject":[],"published":{"date-parts":[[2025,6,22]]},"assertion":[{"value":"2025-06-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}