{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,10]],"date-time":"2026-07-10T20:33:05Z","timestamp":1783715585334,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":104,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,30]],"date-time":"2025-03-30T00:00:00Z","timestamp":1743292800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Strategic Priority Research Program of the Chinese Academy of Sciences","award":["XDB0660100"],"award-info":[{"award-number":["XDB0660100"]}]},{"name":"Semiconductor Research Cooperation"},{"name":"Huawei ZRC Storage Team"},{"name":"ETH Future Computing Laboratory (EFCL)"},{"name":"AI Chip Center for Emerging Smart Systems (ACCESS)"},{"DOI":"10.13039\/501100006374","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62090024, 62222411"],"award-info":[{"award-number":["62090024, 62222411"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Key R&D Program of China","award":["2023YFB4404400"],"award-info":[{"award-number":["2023YFB4404400"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,30]]},"DOI":"10.1145\/3676641.3716009","type":"proceedings-article","created":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T16:47:32Z","timestamp":1743094052000},"page":"766-782","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":27,"title":["PAPI: Exploiting Dynamic Parallelism in Large Language Model Decoding with a Processing-In-Memory-Enabled Computing System"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3054-0617","authenticated-orcid":false,"given":"Yintao","family":"He","sequence":"first","affiliation":[{"name":"SKLP, Institute of Computing Technology, CAS, Beijing, China and University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7393-4504","authenticated-orcid":false,"given":"Haiyu","family":"Mao","sequence":"additional","affiliation":[{"name":"King's College London, London, United Kingdom and ETH Z\u00fcrich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0162-4547","authenticated-orcid":false,"given":"Christina","family":"Giannoula","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Canada, Vector Institute, Toronto, Canada, and ETH Z\u00fcrich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4029-0175","authenticated-orcid":false,"given":"Mohammad","family":"Sadrosadati","sequence":"additional","affiliation":[{"name":"ETH Z\u00fcrich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6514-1571","authenticated-orcid":false,"given":"Juan","family":"G\u00f3mez-Luna","sequence":"additional","affiliation":[{"name":"NVIDIA, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8082-4218","authenticated-orcid":false,"given":"Huawei","family":"Li","sequence":"additional","affiliation":[{"name":"SKLP, Institute of Computing Technology, CAS, Beijing, China and University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0874-814X","authenticated-orcid":false,"given":"Xiaowei","family":"Li","sequence":"additional","affiliation":[{"name":"SKLP, Institute of Computing Technology, CAS, Beijing, China and University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5172-4736","authenticated-orcid":false,"given":"Ying","family":"Wang","sequence":"additional","affiliation":[{"name":"SKLP, Institute of Computing Technology, CAS, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0075-2312","authenticated-orcid":false,"given":"Onur","family":"Mutlu","sequence":"additional","affiliation":[{"name":"ETH Z\u00fcrich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,3,30]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Shaizeen Aga Supreet Jeloka Arun Subramaniyan Satish Narayanasamy David Blaauw and Reetuparna Das. 2017. Compute Caches. In HPCA."},{"key":"e_1_3_2_1_2_1","unstructured":"Amey Agrawal Nitin Kedia Ashish Panwar Jayashree Mohan Nipun Kwatra Bhargav Gulavani Alexey Tumanov and Ramachandran Ramjee. 2024. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In OSDI."},{"key":"e_1_3_2_1_3_1","volume-title":"SARATHI: Efficient LLM Inference by Piggybacking Decodes with Chunked Prefills. arXiv preprint arXiv:2308.16369","author":"Agrawal Amey","year":"2023","unstructured":"Amey Agrawal, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav S Gulavani, and Ramachandran Ramjee. 2023. SARATHI: Efficient LLM Inference by Piggybacking Decodes with Chunked Prefills. arXiv preprint arXiv:2308.16369 (2023)."},{"key":"e_1_3_2_1_4_1","unstructured":"Junwhan Ahn Sungpack Hong Sungjoo Yoo Onur Mutlu and Kiyoung Choi. 2015. A Scalable Processing-in-Memory Accelerator for Parallel Graph Processing. In ISCA."},{"key":"e_1_3_2_1_5_1","unstructured":"Yushi Bai Jiajie Zhang Xin Lv Linzhi Zheng Siqi Zhu Lei Hou Yuxiao Dong Jie Tang and Juanzi Li. 2025. LongWriter: Unleashing 10 000 Word Generation from Long Context LLMs. In ICLR."},{"key":"e_1_3_2_1_6_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language Models are Few-Shot Learners. NeurIPS."},{"key":"e_1_3_2_1_7_1","volume-title":"Accelerating Large Language Model Decoding with Speculative Sampling. arXiv preprint arXiv:2302.01318","author":"Chen Charlie","year":"2023","unstructured":"Charlie Chen, Sebastian Borgeaud, Geoffrey Irving, Jean-Baptiste Lespiau, Laurent Sifre, and John Jumper. 2023. Accelerating Large Language Model Decoding with Speculative Sampling. arXiv preprint arXiv:2302.01318 (2023)."},{"key":"e_1_3_2_1_8_1","first-page":"I","volume":"201","author":"Chen Ke","unstructured":"Ke Chen, Sheng Li, Naveen Muralimanohar, Jung Ho Ahn, Jay B Brockman, and Norman P Jouppi. 2012. CACTI-3DD: Architecture-level Modeling for 3D Die-stacked DRAM Main Memory. In DATE.","journal-title":"Jouppi."},{"key":"e_1_3_2_1_9_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde De Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. 2021. Evaluating Large Language Models Trained on Code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_1_10_1","unstructured":"Tianshi Chen Zidong Du Ninghui Sun Jia Wang Chengyong Wu Yunji Chen and Olivier Temam. 2014. DianNao: A Small-Footprint High-Throughput Accelerator for Ubiquitous Machine-Learning. In ASPLOS."},{"key":"e_1_3_2_1_11_1","volume-title":"Nam Sung Kim, and Jung Ho Ahn","author":"Choi Jaewan","year":"2023","unstructured":"Jaewan Choi, Jaehyun Park, Kwanhee Kyung, Nam Sung Kim, and Jung Ho Ahn. 2023. Unleashing the Potential of PIM: Accelerating Large Batched Inference of Transformer-Based Generative Models. In IEEE CAL."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Jack Choquette and Wish Gandhi. 2020. NVIDIA A100 GPU: Performance & Innovation for GPU Computing. In Hot Chips.","DOI":"10.1109\/HCS49909.2020.9220622"},{"key":"e_1_3_2_1_13_1","volume-title":"Charles Sutton, Sebastian Gehrmann, et al.","author":"Chowdhery Aakanksha","year":"2023","unstructured":"Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, et al. 2023. PaLM: Scaling Language Modeling with Pathways. In JMLR."},{"key":"e_1_3_2_1_14_1","volume-title":"Free Dolly: Introducing the World's First Truly Open Instruction-Tuned LLM","author":"Conover Mike","year":"2023","unstructured":"Mike Conover, Matt Hayes, Ankit Mathur, Jianwei Xie, Jun Wan, Sam Shah, Ali Ghodsi, Patrick Wendell, Matei Zaharia, and Reynold Xin. 2023. Free Dolly: Introducing the World's First Truly Open Instruction-Tuned LLM. In Company Blog of Databricks."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Debendra Das Sharma Robert Blankenship and Daniel Berger. 2024. An Introduction to the Compute Express Link (CXL) Interconnect. In ACM Computing Surveys.","DOI":"10.1145\/3669900"},{"key":"e_1_3_2_1_16_1","unstructured":"Jyotikrishna Dass Shang Wu Huihong Shi Chaojian Li Zhifan Ye Zhongfeng Wang and Yingyan Lin. 2023. ViTALiTy: Unifying Low-rank and Sparse Approximation for Vision Transformer Acceleration with a Linear Taylor Attention. In HPCA."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Peiyan Dong Mengshu Sun Alec Lu Yanyue Xie Kenneth Liu Zhenglun Kong Xin Meng Zhengang Li Xue Lin Zhenman Fang et al. 2023. HeatViT: Hardware-Efficient Adaptive Token Pruning for Vision Transformers. In HPCA.","DOI":"10.1109\/HPCA56546.2023.10071047"},{"key":"e_1_3_2_1_18_1","volume-title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity. In JMLR.","author":"Fedus William","year":"2022","unstructured":"William Fedus, Barret Zoph, and Noam Shazeer. 2022. Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity. In JMLR."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Jo ao Dinis Ferreira Gabriel Falcao Juan G\u00f3mez-Luna Mohammed Alser Lois Orosa Mohammad Sadrosadati Jeremie S Kim Geraldo F Oliveira Taha Shahroodi Anant Nori et al. 2022. pLUTo: Enabling Massively Parallel Computation in DRAM via Lookup Tables. In MICRO.","DOI":"10.1109\/MICRO56248.2022.00067"},{"key":"e_1_3_2_1_20_1","volume-title":"IMI: In-memory Multi-job Inference Acceleration for Large Language Models. In ICPP.","author":"Gao Bin","year":"2024","unstructured":"Bin Gao, Zhehui Wang, Zhuomin He, Tao Luo, Weng-Fai Wong, and Zhi Zhou. 2024. IMI: In-memory Multi-job Inference Acceleration for Large Language Models. In ICPP."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3508041"},{"key":"e_1_3_2_1_22_1","volume-title":"Mohammad Sadrosadati, Juan Gomez Luna, Onur Mutlu, and Gennady Pekhimenko.","author":"Giannoula Christina","year":"2024","unstructured":"Christina Giannoula, Peiming Yang, Ivan Fernandez, Jiacheng Yang, Sankeerth Durvasula, Yu Xin Li, Mohammad Sadrosadati, Juan Gomez Luna, Onur Mutlu, and Gennady Pekhimenko. 2024. PyGim: An Efficient Graph Neural Network Library for Real Processing-In-Memory Architectures. Proceedings of the ACM on Measurement and Analysis of Computing Systems (2024)."},{"key":"e_1_3_2_1_23_1","volume-title":"Ivan Fernandez, Christina Giannoula, Geraldo F Oliveira, and Onur Mutlu.","author":"G\u00f3mez-Luna Juan","year":"2021","unstructured":"Juan G\u00f3mez-Luna, Izzat El Hajj, Ivan Fernandez, Christina Giannoula, Geraldo F Oliveira, and Onur Mutlu. 2021. Benchmarking Memory-Centric Computing Systems: Analysis of Real Processing-in-Memory Hardware. In CUT."},{"key":"e_1_3_2_1_24_1","volume-title":"Ivan Fernandez, Christina Giannoula, Geraldo F Oliveira, and Onur Mutlu.","author":"G\u00f3mez-Luna Juan","year":"2022","unstructured":"Juan G\u00f3mez-Luna, Izzat El Hajj, Ivan Fernandez, Christina Giannoula, Geraldo F Oliveira, and Onur Mutlu. 2022. Benchmarking a New Paradigm: Experimental Analysis and Characterization of a Real Processing-in-Memory System. In IEEE Access."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Juan G\u00f3mez-Luna Yuxin Guo Sylvan Brocard Julien Legriel Remy Cimadomo Geraldo F Oliveira Gagandeep Singh and Onur Mutlu. 2023. Evaluating Machine LearningWorkloads on Memory-Centric Computing Systems. In ISPASS.","DOI":"10.1109\/ISPASS57527.2023.00013"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Cong Guo Jiaming Tang Weiming Hu Jingwen Leng Chen Zhang Fan Yang Yunxin Liu Minyi Guo and Yuhao Zhu. 2023. OliVe: Accelerating Large Language Models via Hardware-friendly Outlier-Victim Pair Quantization. In ISCA.","DOI":"10.1145\/3579371.3589038"},{"key":"e_1_3_2_1_27_1","volume-title":"Seonghak Kim, Young H Oh, Yeonhong Park, Yoonho Song, Jung-Hun Park, Sanghee Lee, Kyoung Park, Jae W Lee, et al.","author":"Ham Tae Jun","year":"2020","unstructured":"Tae Jun Ham, Sung Jun Jung, Seonghak Kim, Young H Oh, Yeonhong Park, Yoonho Song, Jung-Hun Park, Sanghee Lee, Kyoung Park, Jae W Lee, et al. 2020. Atextsuperscript3: Accelerating Attention Mechanisms in Neural Networks with Approximation. In HPCA."},{"key":"e_1_3_2_1_28_1","volume-title":"Soosung Kim, Hyunji Choi, Sung Jun Jung, and Jae W Lee.","author":"Ham Tae Jun","year":"2021","unstructured":"Tae Jun Ham, Yejin Lee, Seong Hoon Seo, Soosung Kim, Hyunji Choi, Sung Jun Jung, and Jae W Lee. 2021. ELSA: Hardware-Software Co-design for Efficient, Lightweight Self-Attention Mechanism in Neural Networks. In ISCA."},{"key":"e_1_3_2_1_29_1","volume-title":"Newton: A DRAM-maker's Accelerator-in-Memory (AiM) Architecture for Machine Learning. In MICRO.","author":"He Mingxuan","year":"2020","unstructured":"Mingxuan He, Choungki Song, Ilkon Kim, Chunseok Jeong, Seho Kim, Il Park, Mithuna Thottethodi, and TN Vijaykumar. 2020a. Newton: A DRAM-maker's Accelerator-in-Memory (AiM) Architecture for Machine Learning. In MICRO."},{"key":"e_1_3_2_1_30_1","unstructured":"Yintao He Ying Wang Cheng Liu Huawei Li and Xiaowei Li. 2021. TARe: Task-Adaptive in-situ ReRAM Computing for Graph Learning. In DAC."},{"key":"e_1_3_2_1_31_1","unstructured":"Yintao He Ying Wang Yongchen Wang Huawei Li and Xiaowei Li. 2019. An Agile Precision-Tunable CNN Accelerator based on ReRAM. In ICCAD."},{"key":"e_1_3_2_1_32_1","unstructured":"Yintao He Ying Wang Xiandong Zhao Huawei Li and Xiaowei Li. 2020b. Towards State-Aware Computation in ReRAM Neural Networks. In DAC."},{"key":"e_1_3_2_1_33_1","unstructured":"Guseul Heo Sangyeop Lee Jaehong Cho Hyunmin Choi Sanghyeon Lee Hyungkyu Ham Gwangsun Kim Divya Mahajan and Jongse Park. 2024. NeuPIMs: NPU-PIM Heterogeneous Acceleration for Batched LLM Inferencing. In ASPLOS."},{"key":"e_1_3_2_1_34_1","volume-title":"DFX: A Low-latency Multi-FPGA Appliance for Accelerating Transformer-based Text Generation. In MICRO.","author":"Hong Seongmin","year":"2022","unstructured":"Seongmin Hong, Seungjae Moon, Junsoo Kim, Sungjae Lee, Minsub Kim, Dongsoo Lee, and Joo-Young Kim. 2022. DFX: A Low-latency Multi-FPGA Appliance for Accelerating Transformer-based Text Generation. In MICRO."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","unstructured":"Bongjoon Hyun Taehun Kim Dongjae Lee and Minsoo Rhu. 2024. Pathfinding Future PIM Architectures by Demystifying a Commercial PIM Technology. In HPCA.","DOI":"10.1109\/HPCA57654.2024.00029"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Hongsun Jang Jaeyong Song Jaewon Jung Jaeyoung Park Youngsok Kim and Jinho Lee. 2024. Smart-Infinity: Fast Large Language Model Training using Near-Storage Processing on a Real System. In HPCA.","DOI":"10.1109\/HPCA57654.2024.00034"},{"key":"e_1_3_2_1_37_1","unstructured":"JEDEC. 2022. High Bandwidth Memory DRAM (HBM3). (2022)."},{"key":"e_1_3_2_1_38_1","volume-title":"PipePIM: Maximizing Computing Unit Utilization in ML-Oriented Digital PIM by Pipelining and Dual Buffering","author":"Jeong Taeyang","unstructured":"Taeyang Jeong and Eui-Young Chung. 2024. PipePIM: Maximizing Computing Unit Utilization in ML-Oriented Digital PIM by Pipelining and Dual Buffering. In IEEE TCAD."},{"key":"e_1_3_2_1_39_1","unstructured":"Norman P Jouppi Cliff Young Nishant Patil David Patterson Gaurav Agrawal Raminder Bajwa Sarah Bates Suresh Bhatia Nan Boden Al Borchers et al. 2017. In-Datacenter Performance Analysis of a Tensor Processing Unit. In ISCA."},{"key":"e_1_3_2_1_40_1","volume-title":"AESPA: Asynchronous Execution Scheme to Exploit Bank-Level Parallelism of Processing-in-Memory. In MICRO.","author":"Kal Hongju","year":"2023","unstructured":"Hongju Kal, Chanyoung Yoo, and Won Woo Ro. 2023. AESPA: Asynchronous Execution Scheme to Exploit Bank-Level Parallelism of Processing-in-Memory. In MICRO."},{"key":"e_1_3_2_1_41_1","volume-title":"FLAT: An Optimized Dataflow for Mitigating Attention Bottlenecks. In ASPLOS.","author":"Kao Sheng-Chun","year":"2023","unstructured":"Sheng-Chun Kao, Suvinay Subramanian, Gaurav Agrawal, Amir Yazdanbakhsh, and Tushar Krishna. 2023. FLAT: An Optimized Dataflow for Mitigating Attention Bottlenecks. In ASPLOS."},{"key":"e_1_3_2_1_42_1","volume-title":"Exploiting Intel\u00ae Advanced Matrix Extensions (AMX) for Large Language Model Inference","author":"Kim Hyungyo","unstructured":"Hyungyo Kim, Gaohan Ye, Nachuan Wang, Amir Yazdanbakhsh, and Nam Sung Kim. 2024. Exploiting Intel\u00ae Advanced Matrix Extensions (AMX) for Large Language Model Inference. In IEEE CAL."},{"key":"e_1_3_2_1_43_1","volume-title":"Ramulator: A Fast and Extensible DRAM Simulator","author":"Kim Yoongu","year":"2015","unstructured":"Yoongu Kim, Weikun Yang, and Onur Mutlu. 2015. Ramulator: A Fast and Extensible DRAM Simulator. In IEEE CAL."},{"key":"e_1_3_2_1_44_1","unstructured":"Hyucksung Kwon Kyungmo Koo Janghyeon Kim Woongkyu Lee Minjae Lee Hyungdeok Lee Yousub Jung Jaehan Park Yosub Song Byeongsu Yang et al. 2024. LoL-PIM: Long-Context LLM Decoding with Scalable DRAM-PIM System. In arXiv preprint arXiv:2412.20166."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Yongkee Kwon Kornijcuk Vladimir Nahsung Kim Woojae Shin Jongsoon Won Minkyu Lee Hyunha Joo Haerang Choi Guhyun Kim Byeongju An et al. 2022. System Architecture and Software Stack for GDDR6-AiM. In HCS.","DOI":"10.1109\/HCS55958.2022.9895629"},{"key":"e_1_3_2_1_46_1","volume-title":"Jaehoon Lee, Sang-Hyuk Kwon, Je Min Ryu, Jong-Pil Son, O Seongil, Hak-Soo Yu, Haesuk Lee, Soo Young Kim, et al.","author":"Kwon Young-Cheon","year":"2021","unstructured":"Young-Cheon Kwon, Suk Han Lee, Jaehoon Lee, Sang-Hyuk Kwon, Je Min Ryu, Jong-Pil Son, O Seongil, Hak-Soo Yu, Haesuk Lee, Soo Young Kim, et al. 2021. 25.4 A 20nm 6GB Function-In-Memory DRAM, Based on HBM2 with a 1.2TFLOPS Programmable Computing Unit Using Bank-Level Parallelism, for Machine Learning Applications. In ISSCC."},{"key":"e_1_3_2_1_47_1","volume-title":"Cost-Effective LLM Accelerator Using Processing in Memory Technology. In 2024 IEEE Symposium on VLSI Technology and Circuits (VLSI Technology and Circuits).","author":"Lee Hyungdeok","year":"2024","unstructured":"Hyungdeok Lee, Guhyun Kim, Dayeon Yun, Ilkon Kim, Yongkee Kwon, and Euicheol Lim. 2024. Cost-Effective LLM Accelerator Using Processing in Memory Technology. In 2024 IEEE Symposium on VLSI Technology and Circuits (VLSI Technology and Circuits)."},{"key":"e_1_3_2_1_48_1","unstructured":"Sukhan Lee Shin-haeng Kang Jaehoon Lee Hyeonsu Kim Eojin Lee Seungwoo Seo Hosang Yoon Seungwon Lee Kyounghwan Lim Hyunsung Shin et al. 2021. Hardware Architecture and Software Stack for PIM Based on Commercial DRAM Technology: Industrial Product. In ISCA."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"S. Lee K. Kim S. Oh J. Park G. Hong D. Ka K. Hwang J. Park K. Kang J. Kim J. Jeon N. Kim Y. Kwon K. Vladimir W. Shin J. Won M. Lee H. Joo et al. 2022. A 1ynm 1.25V 8Gb 16Gb\/s\/pin GDDR6-Based Accelerator-in-Memory Supporting 1TFLOPS MAC Operation and Various Activation Functions for Deep-Learning Applications. In ISSCC.","DOI":"10.1109\/ISSCC42614.2022.9731711"},{"key":"e_1_3_2_1_50_1","unstructured":"Dmitry Lepikhin HyoukJoong Lee Yuanzhong Xu Dehao Chen Orhan Firat Yanping Huang Maxim Krikun Noam Shazeer and Zhifeng Chen. 2021. GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding. In ICLR."},{"key":"e_1_3_2_1_51_1","unstructured":"Yaniv Leviathan Matan Kalman and Yossi Matias. 2023. Fast Inference from Transformers via Speculative Decoding. In ICML."},{"key":"e_1_3_2_1_52_1","unstructured":"Cong Li Zhe Zhou Size Zheng Jiaxi Zhang Yun Liang and Guangyu Sun. 2024b. SpecPIM: Accelerating Speculative Inference on PIM-Enabled System via Architecture-Dataflow Co-Exploration. In ASPLOS."},{"key":"e_1_3_2_1_53_1","volume-title":"ASADI: Accelerating Sparse Attention Using Diagonal-based In-Situ Computing. In HPCA.","author":"Li Huize","year":"2024","unstructured":"Huize Li, Zhaoying Li, Zhenyu Bai, and Tulika Mitra. 2024a. ASADI: Accelerating Sparse Attention Using Diagonal-based In-Situ Computing. In HPCA."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3123939.3123977"},{"key":"e_1_3_2_1_55_1","unstructured":"Zhuohan Li Lianmin Zheng Yinmin Zhong Vincent Liu Ying Sheng Xin Jin Yanping Huang Zhifeng Chen Hao Zhang Joseph E Gonzalez et al. 2023. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In OSDI. 663--679."},{"key":"e_1_3_2_1_56_1","volume-title":"Sanger: A Co-Design Framework for Enabling Sparse Attention using Reconfigurable Architecture. In MICRO.","author":"Lu Liqiang","year":"2021","unstructured":"Liqiang Lu, Yicheng Jin, Hangrui Bi, Zizhang Luo, Peng Li, Tao Wang, and Yun Liang. 2021. Sanger: A Co-Design Framework for Enabling Sparse Attention using Reconfigurable Architecture. In MICRO."},{"key":"e_1_3_2_1_57_1","volume-title":"F Nisa Bostanci, Ataberk Olgun, A Giray Ya\u011flik\u00e7i, and Onur Mutlu.","author":"Luo Haocong","year":"2023","unstructured":"Haocong Luo, Yahya Can Tu\u011frul, F Nisa Bostanci, Ataberk Olgun, A Giray Ya\u011flik\u00e7i, and Onur Mutlu. 2023. Ramulator 2.0: A Modern, Modular, and Extensible DRAM Simulator. In IEEE CAL."},{"key":"e_1_3_2_1_58_1","volume-title":"Accelerating Speculative Decoding using Dynamic Speculation Length. arXiv preprint arXiv:2405.04304","author":"Mamou Jonathan","year":"2024","unstructured":"Jonathan Mamou, Oren Pereg, Daniel Korat, Moshe Berchansky, Nadav Timor, Moshe Wasserblat, and Roy Schwartz. 2024. Accelerating Speculative Decoding using Dynamic Speculation Length. arXiv preprint arXiv:2405.04304 (2024)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/CONECT.2003.1231473"},{"key":"e_1_3_2_1_60_1","unstructured":"Microsoft. 2023. Copilot. https:\/\/github.com\/features\/copilot."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"crossref","unstructured":"David J Miller Philip M Watts and Andrew W Moore. 2009. Motivating Future Interconnects: a Differential Measurement Analysis of PCI Latency. In ANCS.","DOI":"10.1145\/1882486.1882513"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Onur Mutlu Saugata Ghose Juan G\u00f3mez-Luna and Rachata Ausavarungnirun. 2022. A Modern Primer on Processing in Memory. In Emerging computing: from devices to systems: looking beyond Moore and Von Neumann.","DOI":"10.1007\/978-981-16-7487-7_7"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"crossref","unstructured":"Onur Mutlu Ataberk Olgun Geraldo F Oliveira and Ismail Emir Yuksel. 2024. Memory-Centric Computing: Recent Advances in Processing-in-DRAM. In IEDM.","DOI":"10.1109\/IEDM50854.2024.10873410"},{"key":"e_1_3_2_1_64_1","unstructured":"Joel Nider Craig Mustard Andrada Zoltan John Ramsden Larry Liu Jacob Grossbard Mohammad Dashti Romaric Jodin Alexandre Ghiti Jordi Chauzi et al. 2021. A Case Study of Processing-in-Memory in off-the-Shelf Systems. In USENIX ATC 21."},{"key":"e_1_3_2_1_65_1","unstructured":"NVIDIA. 2016. Triton Inference Server. https:\/\/developer.nvidia.com\/triton-inference-server. (2016)."},{"key":"e_1_3_2_1_66_1","unstructured":"NVIDIA. 2017. NVLink. http:\/\/www.nvidia.com\/object\/nvlink.html."},{"key":"e_1_3_2_1_67_1","unstructured":"NVIDIA. 2020. NVIDIA A100 Tensor Core GPU Architecture. White Paper. https:\/\/images.nvidia.com\/aem-dam\/en-zz\/Solutions\/data-center\/nvidia-ampere-architecture-whitepaper.pdf."},{"key":"e_1_3_2_1_68_1","unstructured":"NVIDIA. 2023. DGX A100. https:\/\/resources.nvidia.com\/en-us-dgx-systems\/dgx-ai."},{"key":"e_1_3_2_1_69_1","unstructured":"Hyungjun Oh Kihong Kim Jaemin Kim Sungkyun Kim Junyeol Lee Du-seong Chang and Jiwon Seo. 2024. ExeGPT: Constraint-Aware Resource Scheduling for LLM Inference. In ASPLOS."},{"key":"e_1_3_2_1_70_1","volume-title":"Sora: Creating Video from Text. https:\/\/openai.com\/sora.","author":"AI.","year":"2024","unstructured":"OpenAI. 2024. Sora: Creating Video from Text. https:\/\/openai.com\/sora."},{"key":"e_1_3_2_1_71_1","volume-title":"PIM-AI: A Novel Architecture for High-Efficiency LLM Inference. arXiv preprint arXiv:2411.17309","author":"Ortega Cristobal","year":"2024","unstructured":"Cristobal Ortega, Yann Falevoz, and Renaud Ayrignac. 2024. PIM-AI: A Novel Architecture for High-Efficiency LLM Inference. arXiv preprint arXiv:2411.17309 (2024)."},{"key":"e_1_3_2_1_72_1","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray et al. 2022. Training Language Models to Follow Instructions with Human Feedback. In NeurIPS."},{"key":"e_1_3_2_1_73_1","volume-title":"InstInfer: In-Storage Attention Offloading for Cost-Effective Long-Context LLM Inference. arXiv preprint arXiv:2409.04992","author":"Pan Xiurui","year":"2024","unstructured":"Xiurui Pan, Endian Li, Qiao Li, Shengwen Liang, Yizhou Shan, Ke Zhou, Yingwei Luo, Xiaolin Wang, and Jie Zhang. 2024. InstInfer: In-Storage Attention Offloading for Cost-Effective Long-Context LLM Inference. arXiv preprint arXiv:2409.04992 (2024)."},{"key":"e_1_3_2_1_74_1","volume-title":"Yongsuk Kwon, Nam Sung Kim, and Jung Ho Ahn.","author":"Park Jaehyun","year":"2024","unstructured":"Jaehyun Park, Jaewan Choi, Kwanhee Kyung, Michael Jaemin Kim, Yongsuk Kwon, Nam Sung Kim, and Jung Ho Ahn. 2024a. AttAcc! Unleashing the Power of PIM for Batched Transformer-based Generative Model Inference. In ASPLOS."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"crossref","unstructured":"Sang-Soo Park KyungSoo Kim Jinin So Jin Jung Jonggeon Lee Kyoungwan Woo Nayeon Kim Younghyun Lee Hyungyo Kim Yongsuk Kwon et al. 2024b. An LPDDR-based CXL-PNM Platform for TCO-efficient Inference of Transformer-based Large Language Models. In HPCA.","DOI":"10.1109\/HPCA57654.2024.00078"},{"key":"e_1_3_2_1_76_1","volume-title":"Splitwise: Efficient Generative LLM Inference Using Phase Splitting. In ISCA.","author":"Patel Pratyush","year":"2024","unstructured":"Pratyush Patel, Esha Choukse, Chaojie Zhang, Aashaka Shah, Inigo Goiri, Saeed Maleki, and Ricardo Bianchini. 2024. Splitwise: Efficient Generative LLM Inference Using Phase Splitting. In ISCA."},{"key":"e_1_3_2_1_77_1","volume-title":"DOTA: Detect and Omit Weak Attentions for Scalable Transformer Acceleration. In ASPLOS.","author":"Qu Zheng","year":"2022","unstructured":"Zheng Qu, Liu Liu, Fengbin Tu, Zhaodong Chen, Yufei Ding, and Yuan Xie. 2022. DOTA: Detect and Omit Weak Attentions for Scalable Transformer Acceleration. In ASPLOS."},{"key":"e_1_3_2_1_78_1","volume-title":"Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et al.","author":"Radford Alec","year":"2021","unstructured":"Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et al. 2021. Learning Transferable Visual Models From Natural Language Supervision. In ICML."},{"key":"e_1_3_2_1_79_1","unstructured":"Alec Radford Jeffrey Wu Rewon Child David Luan Dario Amodei Ilya Sutskever et al. 2019. Language Models are Unsupervised Multitask Learners. In OpenAI blog."},{"key":"e_1_3_2_1_80_1","unstructured":"Aditya Ramesh Mikhail Pavlov Gabriel Goh Scott Gray Chelsea Voss Alec Radford Mark Chen and Ilya Sutskever. 2021. Zero-Shot Text-to-Image Generation. In ICML."},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"crossref","unstructured":"Steve Rhyner Haocong Luo Juan G\u00f3mez-Luna Mohammad Sadrosadati Jiawei Jiang Ataberk Olgun Harshita Gupta Ce Zhang and Onur Mutlu. 2024. PIM-Opt: Demystifying Distributed Optimization Algorithms on a Real-World Processing-In-Memory System. In PACT.","DOI":"10.1145\/3656019.3676947"},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"crossref","unstructured":"Robin Rombach Andreas Blattmann Dominik Lorenz Patrick Esser and Bj\u00f6rn Ommer. 2022. High-Resolution Image Synthesis with Latent Diffusion Models. In CVPR.","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"e_1_3_2_1_83_1","volume-title":"Jaewon Park, Yong Ki Kim, Hyochang Kim, Yeong Geol Song, Han-Won Cho, Sunghye Cho, Seung Ho Song, et al.","author":"Ryu Yesin","year":"2023","unstructured":"Yesin Ryu, Sung-Gi Ahn, Jae Hoon Lee, Jaewon Park, Yong Ki Kim, Hyochang Kim, Yeong Geol Song, Han-Won Cho, Sunghye Cho, Seung Ho Song, et al. 2023. A 16 GB 1024 GB\/s HBM3 DRAM With Source-Synchronized Bus Design and On-Die Error Control Scheme for Enhanced RAS Features. In IEEE JSSC."},{"key":"e_1_3_2_1_84_1","volume-title":"Seok Joong Hwang, Yongkee Kwon, Guhyun Kim, Chanwook Park, Ilkon Kim, Jaehan Park, Jeongbin Kim, Woojae Shin, et al.","author":"Seo Minseok","year":"2024","unstructured":"Minseok Seo, Xuan Truong Nguyen, Seok Joong Hwang, Yongkee Kwon, Guhyun Kim, Chanwook Park, Ilkon Kim, Jaehan Park, Jeongbin Kim, Woojae Shin, et al. 2024. IANUS: Integrated Accelerator based on NPU-PIM Unified Memory System. In ASPLOS."},{"key":"e_1_3_2_1_85_1","unstructured":"Tensorflow serving. 2023. Tensorflow. https:\/\/github.com\/tensorflow\/serving."},{"key":"e_1_3_2_1_86_1","volume-title":"Fast Bulk Bitwise AND and OR in DRAM","author":"Seshadri Vivek","unstructured":"Vivek Seshadri, Kevin Hsieh, Amirali Boroum, Donghyuk Lee, Michael A Kozuch, Onur Mutlu, Phillip B Gibbons, and Todd C Mowry. 2015. Fast Bulk Bitwise AND and OR in DRAM. In IEEE CAL."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/3123939.3124544"},{"key":"e_1_3_2_1_88_1","unstructured":"Noam Shazeer Azalia Mirhoseini Krzysztof Maziarz Andy Davis Quoc Le Geoffrey Hinton and Jeff Dean. 2016. Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer. In ICLR."},{"key":"e_1_3_2_1_89_1","volume-title":"NeurIPS Workshop.","author":"Shen Haihao","year":"2023","unstructured":"Haihao Shen, Hanwen Chang, Bo Dong, Yu Luo, and Hengyu Meng. 2023. Efficient LLM inference on CPUs. In NeurIPS Workshop."},{"key":"e_1_3_2_1_90_1","volume-title":"Accelerating LLM Inference with Staged Speculative Decoding. In ICML Workshop.","author":"Spector Benjamin","year":"2023","unstructured":"Benjamin Spector and Chris Re. 2023. Accelerating LLM Inference with Staged Speculative Decoding. In ICML Workshop."},{"key":"e_1_3_2_1_91_1","volume-title":"The Synergy of Speculative Decoding and Batching in Serving Large Language Models. arXiv preprint arXiv:2310.18813","author":"Su Qidong","year":"2023","unstructured":"Qidong Su, Christina Giannoula, and Gennady Pekhimenko. 2023. The Synergy of Speculative Decoding and Batching in Serving Large Language Models. arXiv preprint arXiv:2310.18813 (2023)."},{"key":"e_1_3_2_1_92_1","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar et al. 2023. LLaMA: Open and Efficient Foundation Language Models. arXiv preprint arXiv:2302.13971 (2023)."},{"key":"e_1_3_2_1_93_1","unstructured":"UPMEM. 2018. Introduction to UPMEM PIM. Processing-In-Memory (PIM) on DRAM Accelerator (White Paper)."},{"key":"e_1_3_2_1_94_1","unstructured":"UPMEM. 2020. UPMEM Website. https:\/\/www.upmem.com."},{"key":"e_1_3_2_1_95_1","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan N Gomez \u0141ukasz Kaiser and Illia Polosukhin. 2017. Attention is All You Need. In NeurIPS."},{"key":"e_1_3_2_1_96_1","volume-title":"CTA: Hardware-Software Co-design for Compressed Token Attention Mechanism. In HPCA.","author":"Wang Haoran","year":"2023","unstructured":"Haoran Wang, Haobo Xu, Ying Wang, and Yinhe Han. 2023. CTA: Hardware-Software Co-design for Compressed Token Attention Mechanism. In HPCA."},{"key":"e_1_3_2_1_97_1","doi-asserted-by":"crossref","unstructured":"Hanrui Wang Zhekai Zhang and Song Han. 2021. SpAtten: Efficient Sparse Attention Architecture with Cascade Token and Head Pruning. In HPCA.","DOI":"10.1109\/HPCA51647.2021.00018"},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"crossref","unstructured":"Haojun Xia Zhen Zheng Yuchao Li Donglin Zhuang Zhongzhu Zhou Xiafei Qiu Yong Li Wei Lin and Shuaiwen Leon Song. 2023. Flash-LLM: Enabling Cost-Effective and Highly-Efficient Large Generative Model Inference with Unstructured Sparsity. In VLDB.","DOI":"10.14778\/3626292.3626303"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"crossref","unstructured":"Amir Yazdanbakhsh Ashkan Moradifirouzabadi Zheng Li and Mingu Kang. 2022. Sparse Attention Acceleration with Synergistic In-Memory Pruning and On-Chip Recomputation. In MICRO.","DOI":"10.1109\/MICRO56248.2022.00059"},{"key":"e_1_3_2_1_100_1","unstructured":"Haoran You Zhanyi Sun Huihong Shi Zhongzhi Yu Yang Zhao Yongan Zhang Chaojian Li Baopu Li and Yingyan Lin. 2023. ViTCoD: Vision Transformer Acceleration via Dedicated Algorithm and Accelerator Co-Design. In HPCA."},{"key":"e_1_3_2_1_101_1","volume-title":"Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun.","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. 2022. ORCA: A Distributed Serving System for Transformer-Based Generative Models. In OSDI."},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"crossref","unstructured":"Jun Zhang Jue Wang Huan Li Lidan Shou Ke Chen Gang Chen and Sharad Mehrotra. 2024. Draft & Verify: Lossless Large Language Model Acceleration via Self-Speculative Decoding. In ACL.","DOI":"10.18653\/v1\/2024.acl-long.607"},{"key":"e_1_3_2_1_103_1","doi-asserted-by":"crossref","unstructured":"Minxuan Zhou Weihong Xu Jaeyoung Kang and Tajana Rosing. 2022. TransPIM: A Memory-based Acceleration via Software-Hardware Co-Design for Transformer. In HPCA.","DOI":"10.1109\/HPCA53966.2022.00082"},{"key":"e_1_3_2_1_104_1","unstructured":"Zixuan Zhou Xuefei Ning Ke Hong Tianyu Fu Jiaming Xu Shiyao Li Yuming Lou Luning Wang Zhihang Yuan Xiuhong Li et al. 2024. A Survey on Efficient Inference for Large Language Models. arXiv preprint arXiv:2404.14294 (2024)."}],"event":{"name":"ASPLOS '25: 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","location":"Rotterdam Netherlands","acronym":"ASPLOS '25","sponsor":["SIGPLAN ACM Special Interest Group on Programming Languages","SIGOPS ACM Special Interest Group on Operating Systems","SIGARCH ACM Special Interest Group on Computer Architecture"]},"container-title":["Proceedings of the 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676641.3716009","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3676641.3716009","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T11:06:59Z","timestamp":1755774419000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676641.3716009"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,30]]},"references-count":104,"alternative-id":["10.1145\/3676641.3716009","10.1145\/3676641"],"URL":"https:\/\/doi.org\/10.1145\/3676641.3716009","relation":{},"subject":[],"published":{"date-parts":[[2025,3,30]]},"assertion":[{"value":"2025-03-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}