{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,3]],"date-time":"2026-11-03T13:14:32Z","timestamp":1793711672952,"version":"4.6.1"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770854.3783925","type":"proceedings-article","created":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T12:07:40Z","timestamp":1785499660000},"page":"2482-2493","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["MTGenRec: An Efficient Distributed Training System for Generative Recommendation Models in Meituan"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5483-8322","authenticated-orcid":false,"given":"Yuxiang","family":"Wang","sequence":"first","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6437-7622","authenticated-orcid":false,"given":"Chi","family":"Ma","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2122-915X","authenticated-orcid":false,"given":"Xiao","family":"Yan","sequence":"additional","affiliation":[{"name":"Institute for Math and AI, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-5948-9793","authenticated-orcid":false,"given":"Mincong","family":"Huang","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-6475-3197","authenticated-orcid":false,"given":"Xiaoguang","family":"Li","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-9298-1584","authenticated-orcid":false,"given":"Ruidong","family":"Han","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-3228-2670","authenticated-orcid":false,"given":"Bin","family":"Yin","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-6416-0910","authenticated-orcid":false,"given":"Shangyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2834-8765","authenticated-orcid":false,"given":"Xiang","family":"Li","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7019-140X","authenticated-orcid":false,"given":"Fei","family":"Jiang","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-3768-5759","authenticated-orcid":false,"given":"Lei","family":"Yu","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-4698-8452","authenticated-orcid":false,"given":"Chuan","family":"Liu","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2851-820X","authenticated-orcid":false,"given":"Wei","family":"Lin","sequence":"additional","affiliation":[{"name":"Meituan, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5201-3893","authenticated-orcid":false,"given":"Haowei","family":"Han","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3777-5031","authenticated-orcid":false,"given":"Xiaokai","family":"Zhou","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0059-8458","authenticated-orcid":false,"given":"Bo","family":"Du","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China, National Engineering Research Center for Multimedia Software, Wuhan University, Wuhan, China, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0051-0046","authenticated-orcid":false,"given":"Jiawei","family":"Jiang","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,4,20]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al.","author":"Achiam Josh","year":"2023","unstructured":"Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al., 2023. Gpt-4 technical report. arXiv preprint arXiv:2303.08774 (2023)."},{"key":"e_1_3_2_2_2_1","unstructured":"Austin Appleby. 2008. MurmurHash. https:\/\/github.com\/aappleby\/smhasher. Accessed: 2023-10-15."},{"key":"e_1_3_2_2_3_1","first-page":"1171","article-title":"Linear probing revisited: Tombstones mark the demise of primary clustering. In 2021 IEEE 62nd annual symposium on foundations of computer science (FOCS)","author":"Bender Michael A","year":"2022","unstructured":"Michael A Bender, Bradley C Kuszmaul, and William Kuszmaul. 2022. Linear probing revisited: Tombstones mark the demise of primary clustering. In 2021 IEEE 62nd annual symposium on foundations of computer science (FOCS). IEEE, 1171-1182.","journal-title":"IEEE"},{"key":"e_1_3_2_2_4_1","volume-title":"Flashattention: Fast and memory-efficient exact attention with io-awareness. Advances in neural information processing systems","author":"Dao Tri","year":"2022","unstructured":"Tri Dao, Dan Fu, Stefano Ermon, Atri Rudra, and Christopher R\u00e9. 2022. Flashattention: Fast and memory-efficient exact attention with io-awareness. Advances in neural information processing systems, Vol. 35 (2022), 16344-16359."},{"key":"e_1_3_2_2_5_1","volume-title":"OneRec: Unifying Retrieve and Rank with Generative Recommender and Iterative Preference Alignment. arXiv preprint arXiv:2502.18965","author":"Deng Jiaxin","year":"2025","unstructured":"Jiaxin Deng, Shiyao Wang, Kuo Cai, Lejian Ren, Qigen Hu, Weifeng Ding, Qiang Luo, and Guorui Zhou. 2025. OneRec: Unifying Retrieve and Rank with Generative Recommender and Iterative Preference Alignment. arXiv preprint arXiv:2502.18965 (2025)."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2017.12.012"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330651"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401428"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462976"},{"key":"e_1_3_2_2_10_1","volume-title":"SessionRec: Next session prediction paradigm for generative sequential recommendation. arXiv preprint arXiv:2502.10157","author":"Huang Lei","year":"2025","unstructured":"Lei Huang, Hao Guo, Linzhi Peng, Long Zhang, Xiaoteng Wang, Daoyuan Wang, Shichao Wang, Jinpeng Wang, Lei Wang, and Sheng Chen. 2025. SessionRec: Next session prediction paradigm for generative sequential recommendation. arXiv preprint arXiv:2502.10157 (2025)."},{"key":"e_1_3_2_2_11_1","volume-title":"Gpipe: Efficient training of giant neural networks using pipeline parallelism. Advances in neural information processing systems","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Dehao Chen, Mia Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc V Le, Yonghui Wu, et al., 2019. Gpipe: Efficient training of giant neural networks using pipeline parallelism. Advances in neural information processing systems, Vol. 32 (2019)."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3547387"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56063-7_42"},{"key":"e_1_3_2_2_14_1","volume-title":"Proceedings of the international conference on machine learning","volume":"2279","author":"Jia Zhihao","year":"2018","unstructured":"Zhihao Jia, Sina Lin, Charles R Qi, and Alex Aiken. 2018. Exploring hidden dimensions in parallelizing convolutional neural networks. In Proceedings of the international conference on machine learning, Vol. 2279. 2288."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3326937.3341255"},{"key":"e_1_3_2_2_16_1","volume-title":"A linear speedup analysis of distributed deep learning with sparse and quantized communication. Advances in neural information processing systems","author":"Jiang Peng","year":"2018","unstructured":"Peng Jiang and Gagan Agrawal. 2018. A linear speedup analysis of distributed deep learning with sparse and quantized communication. Advances in neural information processing systems, Vol. 31 (2018)."},{"key":"e_1_3_2_2_17_1","volume-title":"Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980","author":"Kingma Diederik P","year":"2014","unstructured":"Diederik P Kingma and Jimmy Ba. 2014. Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980 (2014)."},{"key":"e_1_3_2_2_18_1","volume-title":"Efficient sequence packing without cross-contamination: Accelerating large language models without impacting performance. arXiv preprint arXiv:2107.02027","author":"Krell Mario Michael","year":"2021","unstructured":"Mario Michael Krell, Matej Kosec, Sergio P Perez, and Andrew Fitzgibbon. 2021. Efficient sequence packing without cross-contamination: Accelerating large language models without impacting performance. arXiv preprint arXiv:2107.02027 (2021)."},{"key":"e_1_3_2_2_19_1","volume-title":"Accelerating recommender systems via hardware'' scale-in''. arXiv preprint arXiv:2009.05230","author":"Krishna Suresh","year":"2020","unstructured":"Suresh Krishna and Ravi Krishna. 2020. Accelerating recommender systems via hardware'' scale-in''. arXiv preprint arXiv:2009.05230 (2020)."},{"key":"e_1_3_2_2_20_1","volume-title":"Laura Wynter, Raghu Kiran Ganti, and Mayank Mishra.","author":"Kundu Achintya","year":"2024","unstructured":"Achintya Kundu, Rhui Dih Lee, Laura Wynter, Raghu Kiran Ganti, and Mayank Mishra. 2024. Enhancing training efficiency using packing with flash attention. arXiv preprint arXiv:2407.09105 (2024)."},{"key":"e_1_3_2_2_21_1","volume-title":"Demystifying workload imbalances in large transformer model training over variable-length sequences. arXiv preprint arXiv:2412.07894","author":"Li Haoyang","year":"2024","unstructured":"Haoyang Li, Fangcheng Fu, Sheng Lin, Hao Ge, Xuanyu Wang, Jiawen Niu, Jie Jiang, and Bin Cui. 2024. Demystifying workload imbalances in large transformer model training over variable-length sequences. arXiv preprint arXiv:2412.07894 (2024)."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539070"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2017.29"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220007"},{"key":"e_1_3_2_2_25_1","volume-title":"HET: Scaling out huge embedding model training via cache-enabled distributed framework. arXiv preprint arXiv:2112.07221","author":"Miao Xupeng","year":"2021","unstructured":"Xupeng Miao, Hailin Zhang, Yining Shi, Xiaonan Nie, Zhi Yang, Yangyu Tao, and Bin Cui. 2021. HET: Scaling out huge embedding model training via cache-enabled distributed framework. arXiv preprint arXiv:2112.07221 (2021)."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"e_1_3_2_2_27_1","volume-title":"Jianyu Huang, Narayanan Sundaraman, Jongsoo Park, Xiaodong Wang, Udit Gupta, Carole-Jean Wu, Alisson G Azzolini, et al.","author":"Naumov Maxim","year":"2019","unstructured":"Maxim Naumov, Dheevatsa Mudigere, Hao-Jun Michael Shi, Jianyu Huang, Narayanan Sundaraman, Jongsoo Park, Xiaodong Wang, Udit Gupta, Carole-Jean Wu, Alisson G Azzolini, et al., 2019. Deep learning recommendation model for personalization and recommendation systems. arXiv preprint arXiv:1906.00091 (2019)."},{"key":"e_1_3_2_2_28_1","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans Ilya Sutskever et al. 2018. Improving language understanding by generative pre-training. (2018)."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"e_1_3_2_2_31_1","volume-title":"Olatunji Ruwase, Shuangyan Yang, Minjia Zhang, Dong Li, and Yuxiong He.","author":"Ren Jie","year":"2021","unstructured":"Jie Ren, Samyam Rajbhandari, Reza Yazdani Aminabadi, Olatunji Ruwase, Shuangyan Yang, Minjia Zhang, Dong Li, and Yuxiong He. 2021. Zero-offload: Democratizing billion-scale model training. In 2021 USENIX annual technical conference. 551-564."},{"key":"e_1_3_2_2_32_1","volume-title":"Megatron-lm: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053","author":"Shoeybi Mohammad","year":"2019","unstructured":"Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper, and Bryan Catanzaro. 2019. Megatron-lm: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053 (2019)."},{"key":"e_1_3_2_2_33_1","first-page":"1","article-title":"Zion: Facebook next-generation large memory training platform. In 2019 31th IEEE hot chips symposium","author":"Smelyanskiy Misha","year":"2019","unstructured":"Misha Smelyanskiy. 2019. Zion: Facebook next-generation large memory training platform. In 2019 31th IEEE hot chips symposium. IEEE, 1-22.","journal-title":"IEEE"},{"key":"e_1_3_2_2_34_1","volume-title":"Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth\u00e9e Lacroix, Baptiste Rozi\u00e8re, Naman Goyal, Eric Hambro, Faisal Azhar, et al., 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"e_1_3_2_2_35_1","volume-title":"Attention is all you need. Advances in neural information processing systems","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems, Vol. 30 (2017)."},{"key":"e_1_3_2_2_36_1","volume-title":"Connor Holmes, Samyam Rajbhandari, Olatunji Ruwase, Feng Yan, Lei Yang, and Yuxiong He.","author":"Wang Guanhua","year":"2023","unstructured":"Guanhua Wang, Heyang Qin, Sam Ade Jacobs, Connor Holmes, Samyam Rajbhandari, Olatunji Ruwase, Feng Yan, Lei Yang, and Yuxiong He. 2023b. Zero: Extremely efficient collective communication for giant model training. arXiv preprint arXiv:2306.10209 (2023)."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3124749.3124754"},{"key":"e_1_3_2_2_38_1","volume-title":"Generative recommendation: Towards next-generation recommender paradigm. arXiv preprint arXiv:2304.03516","author":"Wang Wenjie","year":"2023","unstructured":"Wenjie Wang, Xinyu Lin, Fuli Feng, Xiangnan He, and Tat-Seng Chua. 2023a. Generative recommendation: Towards next-generation recommender paradigm. arXiv preprint arXiv:2304.03516 (2023)."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3523227.3547405"},{"key":"e_1_3_2_2_40_1","volume-title":"BitPipe: Bidirectional interleaved pipeline parallelism for accelerating large models training. arXiv preprint arXiv:2410.19367","author":"Wu Houming","year":"2024","unstructured":"Houming Wu, Ling Chen, and Wenjie Yu. 2024. BitPipe: Bidirectional interleaved pipeline parallelism for accelerating large models training. arXiv preprint arXiv:2410.19367 (2024)."},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00025"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210181"},{"key":"e_1_3_2_2_43_1","volume-title":"An efficient large recommendation model: Towards a resource-optimal scaling law. arXiv preprint arXiv:2502.09888","author":"Xu Songpei","year":"2025","unstructured":"Songpei Xu, Shijia Wang, Da Guo, Xianwen Guo, Qiang Xiao, Fangjian Li, and Chuanjiang Luo. 2025. An efficient large recommendation model: Towards a resource-optimal scaling law. arXiv preprint arXiv:2502.09888 (2025)."},{"key":"e_1_3_2_2_44_1","unstructured":"Bencheng Yan Shilei Liu Zhiyuan Zeng Zihao Wang Yizhen Zhang Yujin Yuan Langming Liu Jiaqi Liu Di Wang Wenbo Su et al. 2025. Unlocking scaling law in industrial recommendation systems with a three-step paradigm based large user model. arXiv preprint arXiv:2502.08309 (2025)."},{"key":"e_1_3_2_2_45_1","unstructured":"Jiaqi Zhai Lucy Liao Xing Liu Yueming Wang Rui Li Xuan Cao Leon Gao Zhaojie Gong Fangda Gu Michael He et al. 2024. Actions speak louder than words: Trillion-parameter sequential transducers for generative recommendations. arXiv preprint arXiv:2402.17152 (2024)."},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE53745.2022.00324"},{"key":"e_1_3_2_2_47_1","first-page":"412","article-title":"Distributed hierarchical gpu parameter server for massive scale deep learning ads systems","volume":"2","author":"Zhao Weijie","year":"2020","unstructured":"Weijie Zhao, Deping Xie, Ronglai Jia, Yulei Qian, Ruiquan Ding, Mingming Sun, and Ping Li. 2020. Distributed hierarchical gpu parameter server for massive scale deep learning ads systems. Proceedings of machine learning and systems, Vol. 2 (2020), 412-428.","journal-title":"Proceedings of machine learning and systems"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3219823"}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3770854.3783925","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,25]],"date-time":"2026-08-25T20:11:34Z","timestamp":1787688694000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770854.3783925"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,20]]},"references-count":48,"alternative-id":["10.1145\/3770854.3783925","10.1145\/3770854"],"URL":"https:\/\/doi.org\/10.1145\/3770854.3783925","relation":{},"subject":[],"published":{"date-parts":[[2026,4,20]]},"assertion":[{"value":"2026-04-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}