{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T23:21:23Z","timestamp":1780356083759,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","funder":[{"name":"IITP","award":["No.RS-2024-00396013, No.2022-0-01037"],"award-info":[{"award-number":["No.RS-2024-00396013, No.2022-0-01037"]}]},{"name":"IITP under the Graduate School of Artificial Intelligence Semiconductor","award":["IITP-2025-RS-2023-00256472"],"award-info":[{"award-number":["IITP-2025-RS-2023-00256472"]}]},{"name":"ETRI","award":["No.RS-2025-02305453"],"award-info":[{"award-number":["No.RS-2025-02305453"]}]},{"DOI":"10.13039\/100003877","name":"IDEC, Korea","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100003877","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,10,18]]},"DOI":"10.1145\/3725843.3756115","type":"proceedings-article","created":{"date-parts":[[2025,10,17]],"date-time":"2025-10-17T17:19:56Z","timestamp":1760721596000},"page":"461-475","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["HLX: A Unified Pipelined Architecture for Optimized Performance of Hybrid Transformer-Mamba Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3228-3725","authenticated-orcid":false,"given":"In-Jun","family":"Jung","sequence":"first","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-3263-0705","authenticated-orcid":false,"given":"Gyeongrok","family":"Yang","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1125-1521","authenticated-orcid":false,"given":"Jaeha","family":"Min","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1099-1496","authenticated-orcid":false,"given":"Joo-Young","family":"Kim","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,10,17]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"Shantanu Acharya Fei Jia and Boris Ginsburg. 2024. Star attention: Efficient llm inference over long sequences. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.17116 (2024)."},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"crossref","unstructured":"Joshua Ainslie James Lee-Thorp Michiel de Jong Yury Zemlyanskiy Federico Lebr\u00f3n and Sumit Sanghai. 2023. GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints. arxiv:https:\/\/arXiv.org\/abs\/2305.13245\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2305.13245","DOI":"10.18653\/v1\/2023.emnlp-main.298"},{"key":"e_1_3_3_1_4_2","unstructured":"Anthropic. 2024. Claude 3.5 Sonnet. https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet."},{"key":"e_1_3_3_1_5_2","unstructured":"Simran Arora Sabri Eyuboglu Michael Zhang Aman Timalsina Silas Alberti Dylan Zinsley James Zou Atri Rudra and Christopher R\u00e9. 2024. Simple linear attention language models balance the recall-throughput tradeoff. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.18668 (2024)."},{"key":"e_1_3_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/2063384.2063400"},{"key":"e_1_3_3_1_7_2","unstructured":"Yonatan Bisk Rowan Zellers Ronan\u00a0Le Bras Jianfeng Gao and Yejin Choi. 2019. PIQA: Reasoning about Physical Commonsense in Natural Language. arxiv:https:\/\/arXiv.org\/abs\/1911.11641\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/1911.11641"},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","unstructured":"Keonhee Cho Heekyung Choi In\u00a0Jun Jung Jisang Oh Tae\u00a0Woo Oh Kiryong Kim Giseok Kim Taemin Choi Changsu Sim Taejoong Song and Seong-Ook Jung. 2022. SRAM Write- and Performance-Assist Cells for Reducing Interconnect Resistance Effects Increased With Technology Scaling. IEEE Journal of Solid-State Circuits 57 4 (2022) 1039\u20131048. 10.1109\/JSSC.2021.3138785","DOI":"10.1109\/JSSC.2021.3138785"},{"key":"e_1_3_3_1_9_2","unstructured":"Peter Clark Isaac Cowhey Oren Etzioni Tushar Khot Ashish Sabharwal Carissa Schoenick and Oyvind Tafjord. 2018. Think you have solved question answering? try arc the ai2 reasoning challenge. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1803.05457 (2018)."},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA57654.2024.00086"},{"key":"e_1_3_3_1_11_2","unstructured":"Tri Dao. 2023. Flashattention-2: Faster attention with better parallelism and work partitioning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.08691 (2023)."},{"key":"e_1_3_3_1_12_2","unstructured":"Tri Dao and Albert Gu. 2024. Transformers are ssms: Generalized models and efficient algorithms through structured state space duality. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.21060 (2024)."},{"key":"e_1_3_3_1_13_2","unstructured":"DeepSeek-AI Aixin Liu Bei Feng Bin Wang Bingxuan Wang Bo Liu Chenggang Zhao Chengqi Dengr Chong Ruan Damai Dai Daya Guo Dejian Yang Deli Chen Dongjie Ji Erhang Li Fangyun Lin Fuli Luo Guangbo Hao Guanting Chen Guowei Li H. Zhang Hanwei Xu Hao Yang Haowei Zhang Honghui Ding Huajian Xin Huazuo Gao Hui Li Hui Qu J.\u00a0L. Cai Jian Liang Jianzhong Guo Jiaqi Ni Jiashi Li Jin Chen Jingyang Yuan Junjie Qiu Junxiao Song Kai Dong Kaige Gao Kang Guan Lean Wang Lecong Zhang Lei Xu Leyi Xia Liang Zhao Liyue Zhang Meng Li Miaojun Wang Mingchuan Zhang Minghua Zhang Minghui Tang Mingming Li Ning Tian Panpan Huang Peiyi Wang Peng Zhang Qihao Zhu Qinyu Chen Qiushi Du R.\u00a0J. Chen R.\u00a0L. Jin Ruiqi Ge Ruizhe Pan Runxin Xu Ruyi Chen S.\u00a0S. Li Shanghao Lu Shangyan Zhou Shanhuang Chen Shaoqing Wu Shengfeng Ye Shirong Ma Shiyu Wang Shuang Zhou Shuiping Yu Shunfeng Zhou Size Zheng T. Wang Tian Pei Tian Yuan Tianyu Sun W.\u00a0L. Xiao Wangding Zeng Wei An Wen Liu Wenfeng Liang Wenjun Gao Wentao Zhang X.\u00a0Q. Li Xiangyue Jin Xianzu Wang Xiao Bi Xiaodong Liu Xiaohan Wang Xiaojin Shen Xiaokang Chen Xiaosha Chen Xiaotao Nie Xiaowen Sun Xiaoxiang Wang Xin Liu Xin Xie Xingkai Yu Xinnan Song Xinyi Zhou Xinyu Yang Xuan Lu Xuecheng Su Y. Wu Y.\u00a0K. Li Y.\u00a0X. Wei Y.\u00a0X. Zhu Yanhong Xu Yanping Huang Yao Li Yao Zhao Yaofeng Sun Yaohui Li Yaohui Wang Yi Zheng Yichao Zhang Yiliang Xiong Yilong Zhao Ying He Ying Tang Yishi Piao Yixin Dong Yixuan Tan Yiyuan Liu Yongji Wang Yongqiang Guo Yuchen Zhu Yuduan Wang Yuheng Zou Yukun Zha Yunxian Ma Yuting Yan Yuxiang You Yuxuan Liu Z.\u00a0Z. Ren Zehui Ren Zhangli Sha Zhe Fu Zhen Huang Zhen Zhang Zhenda Xie Zhewen Hao Zhihong Shao Zhiniu Wen Zhipeng Xu Zhongyu Zhang Zhuoshu Li Zihan Wang Zihui Gu Zilin Li and Ziwei Xie. 2024. DeepSeek-V2: A Strong Economical and Efficient Mixture-of-Experts Language Model. arxiv:https:\/\/arXiv.org\/abs\/2405.04434\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2405.04434"},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"crossref","unstructured":"Stefan Elfwing Eiji Uchibe and Kenji Doya. 2018. Sigmoid-weighted linear units for neural network function approximation in reinforcement learning. Neural networks 107 (2018) 3\u201311.","DOI":"10.1016\/j.neunet.2017.12.012"},{"key":"e_1_3_3_1_15_2","unstructured":"Daniel\u00a0Y Fu Tri Dao Khaled\u00a0K Saab Armin\u00a0W Thomas Atri Rudra and Christopher R\u00e9. 2022. Hungry hungry hippos: Towards language modeling with state space models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2212.14052 (2022)."},{"key":"e_1_3_3_1_16_2","unstructured":"Paolo Glorioso Quentin Anthony Yury Tokpanov Anna Golubeva Vasudev Shyam James Whittington Jonathan Pilault and Beren Millidge. 2024. The Zamba2 Suite: Technical Report. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.15242 (2024)."},{"key":"e_1_3_3_1_17_2","unstructured":"Paolo Glorioso Quentin Anthony Yury Tokpanov James Whittington Jonathan Pilault Adam Ibrahim and Beren Millidge. 2024. Zamba: A compact 7b ssm hybrid model. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.16712 (2024)."},{"key":"e_1_3_3_1_18_2","unstructured":"Google. 2025. Gemini 2.0 is now available to everyone. https:\/\/blog.google\/technology\/google-deepmind\/gemini-model-updates-february-2025."},{"key":"e_1_3_3_1_19_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan Amy Yang Angela Fan Anirudh Goyal Anthony Hartshorn Aobo Yang Archi Mitra Archie Sravankumar Artem Korenev Arthur Hinsvark Arun Rao Aston Zhang Aurelien Rodriguez Austen Gregerson Ava Spataru Baptiste Roziere Bethany Biron Binh Tang Bobbie Chern Charlotte Caucheteux Chaya Nayak Chloe Bi Chris Marra Chris McConnell Christian Keller Christophe Touret Chunyang Wu Corinne Wong Cristian\u00a0Canton Ferrer Cyrus Nikolaidis Damien Allonsius Daniel Song Danielle Pintz Danny Livshits Danny Wyatt David Esiobu Dhruv Choudhary Dhruv Mahajan Diego Garcia-Olano Diego Perino Dieuwke Hupkes Egor Lakomkin Ehab AlBadawy Elina Lobanova Emily Dinan Eric\u00a0Michael Smith Filip Radenovic Francisco Guzm\u00e1n Frank Zhang Gabriel Synnaeve Gabrielle Lee Georgia\u00a0Lewis Anderson Govind Thattai Graeme Nail Gregoire Mialon Guan Pang Guillem Cucurell Hailey Nguyen Hannah Korevaar Hu Xu Hugo Touvron Iliyan Zarov Imanol\u00a0Arrieta Ibarra Isabel Kloumann Ishan Misra Ivan Evtimov Jack Zhang Jade Copet Jaewon Lee Jan Geffert Jana Vranes Jason Park Jay Mahadeokar Jeet Shah Jelmer van\u00a0der Linde Jennifer Billock Jenny Hong Jenya Lee Jeremy Fu Jianfeng Chi Jianyu Huang Jiawen Liu Jie Wang Jiecao Yu Joanna Bitton Joe Spisak Jongsoo Park Joseph Rocca Joshua Johnstun Joshua Saxe Junteng Jia Kalyan\u00a0Vasuden Alwala Karthik Prasad Kartikeya Upasani Kate Plawiak Ke Li Kenneth Heafield Kevin Stone Khalid El-Arini Krithika Iyer Kshitiz Malik Kuenley Chiu Kunal Bhalla Kushal Lakhotia Lauren Rantala-Yeary Laurens van\u00a0der Maaten Lawrence Chen Liang Tan Liz Jenkins Louis Martin Lovish Madaan Lubo Malo Lukas Blecher Lukas Landzaat Luke de Oliveira Madeline Muzzi Mahesh Pasupuleti Mannat Singh Manohar Paluri Marcin Kardas Maria Tsimpoukelli Mathew Oldham Mathieu Rita Maya Pavlova Melanie Kambadur Mike Lewis Min Si Mitesh\u00a0Kumar Singh Mona Hassan Naman Goyal Narjes Torabi Nikolay Bashlykov Nikolay Bogoychev Niladri Chatterji Ning Zhang Olivier Duchenne Onur \u00c7elebi Patrick Alrassy Pengchuan Zhang Pengwei Li Petar Vasic Peter Weng Prajjwal Bhargava Pratik Dubal Praveen Krishnan Punit\u00a0Singh Koura Puxin Xu Qing He Qingxiao Dong Ragavan Srinivasan Raj Ganapathy Ramon Calderer Ricardo\u00a0Silveira Cabral Robert Stojnic Roberta Raileanu Rohan Maheswari Rohit Girdhar Rohit Patel Romain Sauvestre Ronnie Polidoro Roshan Sumbaly Ross Taylor Ruan Silva Rui Hou Rui Wang Saghar Hosseini Sahana Chennabasappa Sanjay Singh Sean Bell Seohyun\u00a0Sonia Kim Sergey Edunov Shaoliang Nie Sharan Narang Sharath Raparthy Sheng Shen Shengye Wan Shruti Bhosale Shun Zhang Simon Vandenhende Soumya Batra Spencer Whitman Sten Sootla Stephane Collot Suchin Gururangan Sydney Borodinsky Tamar Herman Tara Fowler Tarek Sheasha Thomas Georgiou Thomas Scialom Tobias Speckbacher Todor Mihaylov Tong Xiao Ujjwal Karn Vedanuj Goswami Vibhor Gupta Vignesh Ramanathan Viktor Kerkez Vincent Gonguet Virginie Do Vish Vogeti V\u00edtor Albiero Vladan Petrovic Weiwei Chu Wenhan Xiong Wenyin Fu Whitney Meers Xavier Martinet Xiaodong Wang Xiaofang Wang Xiaoqing\u00a0Ellen Tan Xide Xia Xinfeng Xie Xuchao Jia Xuewei Wang Yaelle Goldschlag Yashesh Gaur Yasmine Babaei Yi Wen Yiwen Song Yuchen Zhang Yue Li Yuning Mao Zacharie\u00a0Delpierre Coudert Zheng Yan Zhengxing Chen Zoe Papakipos Aaditya Singh Aayushi Srivastava Abha Jain Adam Kelsey Adam Shajnfeld Adithya Gangidi Adolfo Victoria Ahuva Goldstand Ajay Menon Ajay Sharma Alex Boesenberg Alexei Baevski Allie Feinstein Amanda Kallet Amit Sangani Amos Teo Anam Yunus Andrei Lupu Andres Alvarado Andrew Caples Andrew Gu Andrew Ho Andrew Poulton Andrew Ryan Ankit Ramchandani Annie Dong Annie Franco Anuj Goyal Aparajita Saraf Arkabandhu Chowdhury Ashley Gabriel Ashwin Bharambe Assaf Eisenman Azadeh Yazdan Beau James Ben Maurer Benjamin Leonhardi Bernie Huang Beth Loyd Beto\u00a0De Paola Bhargavi Paranjape Bing Liu Bo Wu Boyu Ni Braden Hancock Bram Wasti Brandon Spence Brani Stojkovic Brian Gamido Britt Montalvo Carl Parker Carly Burton Catalina Mejia Ce Liu Changhan Wang Changkyu Kim Chao Zhou Chester Hu Ching-Hsiang Chu Chris Cai Chris Tindal Christoph Feichtenhofer Cynthia Gao Damon Civin Dana Beaty Daniel Kreymer Daniel Li David Adkins David Xu Davide Testuggine Delia David Devi Parikh Diana Liskovich Didem Foss Dingkang Wang Duc Le Dustin Holland Edward Dowling Eissa Jamil Elaine Montgomery Eleonora Presani Emily Hahn Emily Wood Eric-Tuan Le Erik Brinkman Esteban Arcaute Evan Dunbar Evan Smothers Fei Sun Felix Kreuk Feng Tian Filippos Kokkinos Firat Ozgenel Francesco Caggioni Frank Kanayet Frank Seide Gabriela\u00a0Medina Florez Gabriella Schwarz Gada Badeer Georgia Swee Gil Halpern Grant Herman Grigory Sizov Guangyi Zhang Guna Lakshminarayanan Hakan Inan Hamid Shojanazeri Han Zou Hannah Wang Hanwen Zha Haroun Habeeb Harrison Rudolph Helen Suk Henry Aspegren Hunter Goldman Hongyuan Zhan Ibrahim Damlaj Igor Molybog Igor Tufanov Ilias Leontiadis Irina-Elena Veliche Itai Gat Jake Weissman James Geboski James Kohli Janice Lam Japhet Asher Jean-Baptiste Gaya Jeff Marcus Jeff Tang Jennifer Chan Jenny Zhen Jeremy Reizenstein Jeremy Teboul Jessica Zhong Jian Jin Jingyi Yang Joe Cummings Jon Carvill Jon Shepard Jonathan McPhie Jonathan Torres Josh Ginsburg Junjie Wang Kai Wu Kam\u00a0Hou U Karan Saxena Kartikay Khandelwal Katayoun Zand Kathy Matosich Kaushik Veeraraghavan Kelly Michelena Keqian Li Kiran Jagadeesh Kun Huang Kunal Chawla Kyle Huang Lailin Chen Lakshya Garg Lavender A Leandro Silva Lee Bell Lei Zhang Liangpeng Guo Licheng Yu Liron Moshkovich Luca Wehrstedt Madian Khabsa Manav Avalani Manish Bhatt Martynas Mankus Matan Hasson Matthew Lennie Matthias Reso Maxim Groshev Maxim Naumov Maya Lathi Meghan Keneally Miao Liu Michael\u00a0L. Seltzer Michal Valko Michelle Restrepo Mihir Patel Mik Vyatskov Mikayel Samvelyan Mike Clark Mike Macey Mike Wang Miquel\u00a0Jubert Hermoso Mo Metanat Mohammad Rastegari Munish Bansal Nandhini Santhanam Natascha Parks Natasha White Navyata Bawa Nayan Singhal Nick Egebo Nicolas Usunier Nikhil Mehta Nikolay\u00a0Pavlovich Laptev Ning Dong Norman Cheng Oleg Chernoguz Olivia Hart Omkar Salpekar Ozlem Kalinli Parkin Kent Parth Parekh Paul Saab Pavan Balaji Pedro Rittner Philip Bontrager Pierre Roux Piotr Dollar Polina Zvyagina Prashant Ratanchandani Pritish Yuvraj Qian Liang Rachad Alao Rachel Rodriguez Rafi Ayub Raghotham Murthy Raghu Nayani Rahul Mitra Rangaprabhu Parthasarathy Raymond Li Rebekkah Hogan Robin Battey Rocky Wang Russ Howes Ruty Rinott Sachin Mehta Sachin Siby Sai\u00a0Jayesh Bondu Samyak Datta Sara Chugh Sara Hunt Sargun Dhillon Sasha Sidorov Satadru Pan Saurabh Mahajan Saurabh Verma Seiji Yamamoto Sharadh Ramaswamy Shaun Lindsay Shaun Lindsay Sheng Feng Shenghao Lin Shengxin\u00a0Cindy Zha Shishir Patil Shiva Shankar Shuqiang Zhang Shuqiang Zhang Sinong Wang Sneha Agarwal Soji Sajuyigbe Soumith Chintala Stephanie Max Stephen Chen Steve Kehoe Steve Satterfield Sudarshan Govindaprasad Sumit Gupta Summer Deng Sungmin Cho Sunny Virk Suraj Subramanian Sy Choudhury Sydney Goldman Tal Remez Tamar Glaser Tamara Best Thilo Koehler Thomas Robinson Tianhe Li Tianjun Zhang Tim Matthews Timothy Chou Tzook Shaked Varun Vontimitta Victoria Ajayi Victoria Montanez Vijai Mohan Vinay\u00a0Satish Kumar Vishal Mangla Vlad Ionescu Vlad Poenaru Vlad\u00a0Tiberiu Mihailescu Vladimir Ivanov Wei Li Wenchen Wang Wenwen Jiang Wes Bouaziz Will Constable Xiaocheng Tang Xiaojian Wu Xiaolan Wang Xilun Wu Xinbo Gao Yaniv Kleinman Yanjun Chen Ye Hu Ye Jia Ye Qi Yenda Li Yilin Zhang Ying Zhang Yossi Adi Youngjin Nam Yu Wang Yu Zhao Yuchen Hao Yundi Qian Yunlu Li Yuzi He Zach Rait Zachary DeVito Zef Rosnbrick Zhaoduo Wen Zhenyu Yang Zhiwei Zhao and Zhiyu Ma. 2024. The Llama 3 Herd of Models. arxiv:https:\/\/arXiv.org\/abs\/2407.21783\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_3_1_20_2","unstructured":"Albert Gu and Tri Dao. 2023. Mamba: Linear-time sequence modeling with selective state spaces. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2312.00752 (2023)."},{"key":"e_1_3_3_1_21_2","unstructured":"Albert\u00a0Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra\u00a0Singh Chaplot Diego de\u00a0las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio\u00a0Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven\u00a0Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William\u00a0El Sayed. 2023. Mistral 7B. arxiv:https:\/\/arXiv.org\/abs\/2310.06825\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2310.06825"},{"key":"e_1_3_3_1_22_2","unstructured":"Albert\u00a0Q. Jiang Alexandre Sablayrolles Antoine Roux Arthur Mensch Blanche Savary Chris Bamford Devendra\u00a0Singh Chaplot Diego de\u00a0las Casas Emma\u00a0Bou Hanna Florian Bressand Gianna Lengyel Guillaume Bour Guillaume Lample L\u00e9lio\u00a0Renard Lavaud Lucile Saulnier Marie-Anne Lachaux Pierre Stock Sandeep Subramanian Sophia Yang Szymon Antoniak Teven\u00a0Le Scao Th\u00e9ophile Gervet Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William\u00a0El Sayed. 2024. Mixtral of Experts. arxiv:https:\/\/arXiv.org\/abs\/2401.04088\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2401.04088"},{"key":"e_1_3_3_1_23_2","unstructured":"Shengyu\u00a0Liu Jiashi\u00a0Li. 2025. FlashMLA: Efficient MLA decoding kernels. https:\/\/github.com\/deepseek-ai\/FlashMLA."},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA52012.2021.00010"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"crossref","unstructured":"Norman\u00a0P Jouppi Doe\u00a0Hyun Yoon George Kurian Sheng Li Nishant Patil James Laudon Cliff Young and David Patterson. 2020. A domain-specific supercomputer for training deep neural networks. Commun. ACM 63 7 (2020) 67\u201378.","DOI":"10.1145\/3360307"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO61859.2024.00106"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3676536.3676798"},{"key":"e_1_3_3_1_28_2","unstructured":"Opher Lieber Barak Lenz Hofit Bata Gal Cohen Jhonathan Osin Itay Dalmedigos Erez Safahi Shaked Meirom Yonatan Belinkov Shai Shalev-Shwartz Omri Abend Raz Alon Tomer Asida Amir Bergman Roman Glozman Michael Gokhman Avashalom Manevich Nir Ratner Noam Rozen Erez Shwartz Mor Zusman and Yoav Shoham. 2024. Jamba: A Hybrid Transformer-Mamba Language Model. arxiv:https:\/\/arXiv.org\/abs\/2403.19887\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2403.19887"},{"key":"e_1_3_3_1_29_2","unstructured":"Hanxiao Liu Zihang Dai David So and Quoc\u00a0V Le. 2021. Pay attention to mlps. Advances in neural information processing systems 34 (2021) 9204\u20139215."},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS57955.2024.00064"},{"key":"e_1_3_3_1_31_2","unstructured":"Stephen Merity Caiming Xiong James Bradbury and Richard Socher. 2016. Pointer sentinel mixture models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1609.07843 (2016)."},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"crossref","unstructured":"Todor Mihaylov Peter Clark Tushar Khot and Ashish Sabharwal. 2018. Can a suit of armor conduct electricity? a new dataset for open book question answering. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1809.02789 (2018).","DOI":"10.18653\/v1\/D18-1260"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.1109\/IEDM45741.2023.10413890"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO61859.2024.00107"},{"key":"e_1_3_3_1_35_2","unstructured":"NVIDIA. 2025. NVIDIA A100 Tensor Core GPU. https:\/\/www.nvidia.com\/en-us\/data-center\/a100\/."},{"key":"e_1_3_3_1_36_2","unstructured":"NVIDIA. 2025. NVIDIA A100 Tensor Core GPU Architecture. https:\/\/images.nvidia.com\/aem-dam\/en-zz\/Solutions\/data-center\/nvidia-ampere-architecture-whitepaper.pdf."},{"key":"e_1_3_3_1_37_2","unstructured":"NVIDIA. 2025. NVIDIA H100 Tensor Core GPU Architecture. https:\/\/www.nvidia.com\/en-us\/data-center\/h100\/."},{"key":"e_1_3_3_1_38_2","unstructured":"NVIDIA. 2025. NVIDIA H100 Tensor Core GPU Architecture. https:\/\/resources.nvidia.com\/en-us-hopper-architecture\/nvidia-h100-tensor-c."},{"key":"e_1_3_3_1_39_2","unstructured":"NVIDIA. 2025. NVIDIA Nsight Compute. https:\/\/developer.nvidia.com\/nsight-compute."},{"key":"e_1_3_3_1_40_2","unstructured":"NVIDIA. 2025. NVIDIA Nsight Systems. https:\/\/developer.nvidia.com\/nsight-systems."},{"key":"e_1_3_3_1_41_2","unstructured":"NVIDIA. 2025. NVIDIA System Management Interface SMI. https:\/\/developer.nvidia.com\/system-management-interface."},{"key":"e_1_3_3_1_42_2","unstructured":"OpenAI. 2024. OpenAI Documentation - Models. https:\/\/platform.openai.com\/docs\/models\/overview."},{"key":"e_1_3_3_1_43_2","unstructured":"Denis Paperno Germ\u00e1n Kruszewski Angeliki Lazaridou Quan\u00a0Ngoc Pham Raffaella Bernardi Sandro Pezzelle Marco Baroni Gemma Boleda and Raquel Fern\u00e1ndez. 2016. The LAMBADA dataset: Word prediction requiring a broad discourse context. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1606.06031 (2016)."},{"key":"e_1_3_3_1_44_2","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO61859.2024.00100"},{"key":"e_1_3_3_1_45_2","unstructured":"PyTorch. 2025. PyTorch Documentation. https:\/\/pytorch.org\/docs\/stable\/index.html."},{"key":"e_1_3_3_1_46_2","unstructured":"Liliang Ren Yang Liu Yadong Lu Yelong Shen Chen Liang and Weizhu Chen. 2024. Samba: Simple hybrid state space models for efficient unlimited context language modeling. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.07522 (2024)."},{"key":"e_1_3_3_1_47_2","doi-asserted-by":"crossref","unstructured":"Keisuke Sakaguchi Ronan\u00a0Le Bras Chandra Bhagavatula and Yejin Choi. 2021. Winogrande: An adversarial winograd schema challenge at scale. Commun. ACM 64 9 (2021) 99\u2013106.","DOI":"10.1145\/3474381"},{"key":"e_1_3_3_1_48_2","doi-asserted-by":"crossref","unstructured":"Jay Shah Ganesh Bikshandi Ying Zhang Vijay Thakkar Pradeep Ramani and Tri Dao. 2025. Flashattention-3: Fast and accurate attention with asynchrony and low-precision. Advances in Neural Information Processing Systems 37 (2025) 68658\u201368685.","DOI":"10.52202\/079017-2193"},{"key":"e_1_3_3_1_49_2","unstructured":"Synopsys. 2025. Synopsys Design Compiler. https:\/\/www.synopsys.com\/implementation-and-signoff\/rtl-synthesis-test\/dc-ultra.html."},{"key":"e_1_3_3_1_50_2","unstructured":"Jamba Team Barak Lenz Alan Arazi Amir Bergman Avshalom Manevich Barak Peleg Ben Aviram Chen Almagor Clara Fridman Dan Padnos Daniel Gissin Daniel Jannai Dor Muhlgay Dor Zimberg Edden\u00a0M Gerber Elad Dolev Eran Krakovsky Erez Safahi Erez Schwartz Gal Cohen Gal Shachaf Haim Rozenblum Hofit Bata Ido Blass Inbal Magar Itay Dalmedigos Jhonathan Osin Julie Fadlon Maria Rozman Matan Danos Michael Gokhman Mor Zusman Naama Gidron Nir Ratner Noam Gat Noam Rozen Oded Fried Ohad Leshno Omer Antverg Omri Abend Opher Lieber Or Dagan Orit Cohavi Raz Alon Ro\u2019i Belson Roi Cohen Rom Gilad Roman Glozman Shahar Lev Shaked Meirom Tal Delbari Tal Ness Tomer Asida Tom\u00a0Ben Gal Tom Braude Uriya Pumerantz Yehoshua Cohen Yonatan Belinkov Yuval Globerson Yuval\u00a0Peleg Levy and Yoav Shoham. 2024. Jamba-1.5: Hybrid Transformer-Mamba Models at Scale. arxiv:https:\/\/arXiv.org\/abs\/2408.12570\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2408.12570"},{"key":"e_1_3_3_1_51_2","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan\u00a0N Gomez \u0141ukasz Kaiser and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_1_52_2","unstructured":"Roger Waleffe Wonmin Byeon Duncan Riach Brandon Norick Vijay Korthikanti Tri Dao Albert Gu Ali Hatamizadeh Sudhakar Singh Deepak Narayanan Garvit Kulshreshtha Vartika Singh Jared Casper Jan Kautz Mohammad Shoeybi and Bryan Catanzaro. 2024. An empirical study of mamba-based language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.07887 (2024)."},{"key":"e_1_3_3_1_53_2","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO61859.2024.00093"},{"key":"e_1_3_3_1_54_2","unstructured":"Jingyang Yuan Huazuo Gao Damai Dai Junyu Luo Liang Zhao Zhengyan Zhang Zhenda Xie Y.\u00a0X. Wei Lean Wang Zhiping Xiao Yuqing Wang Chong Ruan Ming Zhang Wenfeng Liang and Wangding Zeng. 2025. Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention. arxiv:https:\/\/arXiv.org\/abs\/2502.11089\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2502.11089"},{"key":"e_1_3_3_1_55_2","doi-asserted-by":"crossref","unstructured":"Rowan Zellers Ari Holtzman Yonatan Bisk Ali Farhadi and Yejin Choi. 2019. Hellaswag: Can a machine really finish your sentence?arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1905.07830 (2019).","DOI":"10.18653\/v1\/P19-1472"}],"event":{"name":"MICRO 2025: 58th IEEE\/ACM International Symposium on Microarchitecture","location":"Seoul Korea","acronym":"MICRO 2025","sponsor":["SIGMICRO ACM Special Interest Group on Microarchitectural Research and Processing"]},"container-title":["Proceedings of the 58th IEEE\/ACM International Symposium on Microarchitecture"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3725843.3756115","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,26]],"date-time":"2026-01-26T21:41:36Z","timestamp":1769463696000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3725843.3756115"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,17]]},"references-count":54,"alternative-id":["10.1145\/3725843.3756115","10.1145\/3725843"],"URL":"https:\/\/doi.org\/10.1145\/3725843.3756115","relation":{},"subject":[],"published":{"date-parts":[[2025,10,17]]},"assertion":[{"value":"2025-10-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}