{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T03:21:32Z","timestamp":1780716092485,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":89,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,27]],"date-time":"2024-04-27T00:00:00Z","timestamp":1714176000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,27]]},"DOI":"10.1145\/3620666.3651380","type":"proceedings-article","created":{"date-parts":[[2024,4,24]],"date-time":"2024-04-24T12:08:21Z","timestamp":1713960501000},"page":"722-737","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":97,"title":["NeuPIMs: NPU-PIM Heterogeneous Acceleration for Batched LLM Inferencing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-3179-8541","authenticated-orcid":false,"given":"Guseul","family":"Heo","sequence":"first","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-7645-6503","authenticated-orcid":false,"given":"Sangyeop","family":"Lee","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Korea, South ? Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8973-5164","authenticated-orcid":false,"given":"Jaehong","family":"Cho","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4116-3896","authenticated-orcid":false,"given":"Hyunmin","family":"Choi","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7060-6315","authenticated-orcid":false,"given":"Sanghyeon","family":"Lee","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Korea, South ? Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4989-2834","authenticated-orcid":false,"given":"Hyungkyu","family":"Ham","sequence":"additional","affiliation":[{"name":"POSTECH, Pohang, Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5749-5794","authenticated-orcid":false,"given":"Gwangsun","family":"Kim","sequence":"additional","affiliation":[{"name":"POSTECH, Pohang, Korea, South ? Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-8184-0528","authenticated-orcid":false,"given":"Divya","family":"Mahajan","sequence":"additional","affiliation":[{"name":"Georgia Institute of Technology, Atlanta, GA, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6629-449X","authenticated-orcid":false,"given":"Jongse","family":"Park","sequence":"additional","affiliation":[{"name":"KAIST, Daejeon, Korea, South ? Republic of Korea"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,4,27]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Nvidia Tensor RT 4.0. https:\/\/developer.nvidia.com\/tensorrt."},{"key":"e_1_3_2_1_2_1","volume-title":"https:\/\/github.com\/huggingface\/transformers\/tree\/main","year":"2022","unstructured":"HuggingFace. https:\/\/github.com\/huggingface\/transformers\/tree\/main, 2022."},{"key":"e_1_3_2_1_3_1","volume-title":"https:\/\/github.com\/PSAL-POSTECH\/ONNXim","author":"Simulator Xim NPU","year":"2024","unstructured":"ONNXim NPU Simulator. https:\/\/github.com\/PSAL-POSTECH\/ONNXim, 2024."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.14778\/3485450.3485462"},{"key":"e_1_3_2_1_5_1","volume-title":"Divya Mahajan, and Prashant J Nair. Heterogeneous Acceleration Pipeline for Recommendation System Training. arXiv preprint arXiv:2204.05436","author":"Adnan Muhammad","year":"2022","unstructured":"Muhammad Adnan, Yassaman Ebrahimzadeh Maboud, Divya Mahajan, and Prashant J Nair. Heterogeneous Acceleration Pipeline for Recommendation System Training. arXiv preprint arXiv:2204.05436, 2022."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2749469.2750386"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2749469.2750385"},{"key":"e_1_3_2_1_8_1","first-page":"06","article-title":"Enabling Efficient Inference of Transformer Models at Unprecedented Scale. Technical report","author":"Aminabadi Reza Yazdani","year":"2022","unstructured":"Reza Yazdani Aminabadi, Samyam Rajbhandari, Minjia Zhang, Ammar Ahmad Awan, Cheng Li, Du Li, Elton Zheng, Jeff Rasley, Shaden Smith, Olatunji Ruwase, and Yuxiong He. DeepSpeed Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale. Technical report, Microsoft, 06 2022.","journal-title":"Microsoft"},{"key":"e_1_3_2_1_9_1","unstructured":"Rohan Anil Andrew M. Dai Orhan Firat Melvin Johnson Dmitry Lepikhin Alexandre Passos Siamak Shakeri Emanuel Taropa Paige Bailey Zhifeng Chen Eric Chu Jonathan H. Clark Laurent El Shafey Yanping Huang Kathy Meier-Hellstern Gaurav Mishra Erica Moreira Mark Omernick Kevin Robinson Sebastian Ruder Yi Tay Kefan Xiao Yuanzhong Xu Yujing Zhang Gustavo Hernandez Abrego Junwhan Ahn Jacob Austin Paul Barham Jan Botha James Bradbury Siddhartha Brahma Kevin Brooks Michele Catasta Yong Cheng Colin Cherry Christopher A. Choquette-Choo Aakanksha Chowdhery Cl\u00e9ment Crepy Shachi Dave Mostafa Dehghani Sunipa Dev Jacob Devlin Mark D\u00edaz Nan Du Ethan Dyer Vlad Feinberg Fangxiaoyu Feng Vlad Fienber Markus Freitag Xavier Garcia Sebastian Gehrmann Lucas Gonzalez Guy Gur-Ari Steven Hand Hadi Hashemi Le Hou Joshua Howland Andrea Hu Jeffrey Hui Jeremy Hurwitz Michael Isard Abe Ittycheriah Matthew Jagielski Wenhao Jia Kathleen Kenealy Maxim Krikun Sneha Kudugunta Chang Lan Katherine Lee Benjamin Lee Eric Li Music Li Wei Li YaGuang Li Jian Li Hyeontaek Lim Hanzhao Lin Zhongtao Liu Frederick Liu Marcello Maggioni Aroma Mahendru Joshua Maynez Vedant Misra Maysam Moussalem Zachary Nado John Nham Eric Ni Andrew Nystrom Alicia Parrish Marie Pellat Martin Polacek Alex Polozov Reiner Pope Siyuan Qiao Emily Reif Bryan Richter Parker Riley Alex Castro Ros Aurko Roy Brennan Saeta Rajkumar Samuel Renee Shelby Ambrose Slone Daniel Smilkov David R. So Daniel Sohn Simon Tokumine Dasha Valter Vijay Vasudevan Kiran Vodrahalli Xuezhi Wang Pidong Wang Zirui Wang Tao Wang John Wieting Yuhuai Wu Kelvin Xu Yunhan Xu Linting Xue Pengcheng Yin Jiahui Yu Qiao Zhang Steven Zheng Ce Zheng Weikang Zhou Denny Zhou Slav Petrov and Yonghui Wu. PaLM 2 Technical Report 2023."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2016.7783753"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3085572"},{"key":"e_1_3_2_1_12_1","volume-title":"Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, and Samuel Weinbach. GPT-NeoX-20B: An Open-Source Autoregressive Language Model","author":"Black Sid","year":"2022","unstructured":"Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, Michael Pieler, USVSN Sai Prashanth, Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, and Samuel Weinbach. GPT-NeoX-20B: An Open-Source Autoregressive Language Model, 2022."},{"key":"e_1_3_2_1_13_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. Evaluating Large Language Models Trained on Code 2021."},{"key":"e_1_3_2_1_14_1","volume-title":"PIMCloud: QoS-Aware Resource Management of Latency-Critical Applications in Clouds with Processing-in-Memory. In HPCA","author":"Chen Shuang","year":"2022","unstructured":"Shuang Chen, Yi Jiang, Christina Delimitrou, and Jos\u00e9 F. Mart\u00ednez. PIMCloud: QoS-Aware Resource Management of Latency-Critical Applications in Clouds with Processing-in-Memory. In HPCA, 2022."},{"key":"e_1_3_2_1_15_1","volume-title":"SC","author":"Cho Benjamin Y.","year":"2021","unstructured":"Benjamin Y. Cho, Jeageun Jung, and Mattan Erez. Accelerating bandwidth-bound deep learning inference with main-memory accelerators. In SC, 2021."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/LCA.2023.3305386"},{"key":"e_1_3_2_1_17_1","volume-title":"https:\/\/onnxruntime.ai\/","author":"Runtime ONNX","year":"2021","unstructured":"ONNX Runtime developers. ONNX Runtime. https:\/\/onnxruntime.ai\/, 2021."},{"key":"e_1_3_2_1_18_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In arXiv","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In arXiv, 2018."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3352460.3358260"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3547353.3522661"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3489048.3522661"},{"key":"e_1_3_2_1_22_1","volume-title":"ISVLSI","author":"Giannoula Christina","year":"2022","unstructured":"Christina Giannoula, Ivan Fernandez, Juan G\u00f3mez-Luna, Nectarios Koziris, Georgios Goumas, and Onur Mutlu. SparseP: Efficient Sparse Matrix Vector Multiplication on Real Processing-In-Memory Architectures. In ISVLSI, 2022."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISVLSI54635.2022.00064"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2022.3174101"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO50266.2020.00040"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCAD57390.2023.10323637"},{"key":"e_1_3_2_1_27_1","volume-title":"Training Compute-Optimal Large Language Models","author":"Hoffmann Jordan","year":"2022","unstructured":"Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Trevor Cai, Eliza Rutherford, Diego de Las Casas, Lisa Anne Hendricks, Johannes Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van den Driessche, Bogdan Damoc, Aurelia Guy, Simon Osindero, Karen Simonyan, Erich Elsen, Jack W. Rae, Oriol Vinyals, and Laurent Sifre. Training Compute-Optimal Large Language Models, 2022."},{"key":"e_1_3_2_1_28_1","volume-title":"NeurIPS","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Dehao Chen, Mia Xu Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc V. Le, Yonghui Wu, and Zhifeng Chen. GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism. In NeurIPS, 2019."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA57654.2024.00029"},{"key":"e_1_3_2_1_30_1","volume-title":"ISCA","author":"Imani Mohsen","year":"2019","unstructured":"Mohsen Imani, Saransh Gupta, Yeseong Kim, and Tajana Rosing. FloatPIM: In-Memory Acceleration of Deep Neural Network Training with High Precision. In ISCA, 2019."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO50266.2020.00039"},{"key":"e_1_3_2_1_32_1","volume-title":"Calculating memory system power for DDR3. Micron Designline, 13(1)","author":"Janzen J","year":"2008","unstructured":"J Janzen. Calculating memory system power for DDR3. Micron Designline, 13(1), 2008."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3639046"},{"key":"e_1_3_2_1_34_1","volume-title":"ISCA","author":"Jouppi Norm","year":"2023","unstructured":"Norm Jouppi, George Kurian, Sheng Li, Peter Ma, Rahul Nagarajan, Lifeng Nai, Nishant Patil, Suvinay Subramanian, Andy Swing, Brian Towles, Clifford Young, Xiang Zhou, Zongwei Zhou, and David A Patterson. TPU v4: An Optically Reconfigurable Supercomputer for Machine Learning with Hardware Support for Embeddings. In ISCA, 2023."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3613424.3614314"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490422.3502355"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA45697.2020.00070"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/PACT58117.2023.00018"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA51647.2021.00030"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/HCS52781.2021.9567191"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_2_1_43_1","unstructured":"Young-Cheon Kwon Suk Han Lee Jaehoon Lee Sang-Hyuk Kwon Je Min Ryu Jong-Pil Son O Seongil Hak-Soo Yu Haesuk Lee Soo Young Kim Youngmin Cho Jin Guk Kim Jongyoon Choi Hyun-Sung Shin Jin Kim BengSeng Phuah HyoungMin Kim Myeong Jun Song Ahn Choi Daeho Kim SooYoung Kim Eun-Bong Kim David Wang Shinhaeng Kang Yuhwan Ro Seungwoo Seo JoonHo Song Jaeyoun Youn Kyomin Sohn and Nam Sung Kim. 25.4 A 20nm 6GB Function-In-Memory DRAM Based on HBM2 with a 1.2TFLOPS Programmable Computing Unit Using Bank-Level Parallelism for Machine Learning Applications. In ISSCC 2021."},{"key":"e_1_3_2_1_44_1","volume-title":"MICRO","author":"Kwon Youngeun","year":"2019","unstructured":"Youngeun Kwon, Yunjae Lee, and Minsoo Rhu. TensorDIMM: A Practical Near-Memory Processing Architecture for Embeddings and Tensor Operations in Deep Learning. In MICRO, 2019."},{"key":"e_1_3_2_1_45_1","volume-title":"ISCA","author":"Kwon Youngeun","year":"2022","unstructured":"Youngeun Kwon and Minsoo Rhu. Training Personalized Recommendation Systems from (GPU) Scratch: Look Forward Not Backwards. In ISCA, 2022."},{"key":"e_1_3_2_1_46_1","volume-title":"DATE","author":"Laguna Ann Franchesca","year":"2021","unstructured":"Ann Franchesca Laguna, Arman Kazemi, Michael Niemier, and X. Sharon Hu. In-Memory Computing based Accelerator for Transformer Networks for Long Sequences. In DATE, 2021."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/HCS52781.2021.9566881"},{"key":"e_1_3_2_1_48_1","volume-title":"MAC Operation and Various Activation Functions for Deep-Learning Applications. In 2022 IEEE International Solid-State Circuits Conference (ISSCC)","author":"Lee Seongju","year":"2022","unstructured":"Seongju Lee, Kyuyoung Kim, Sanghoon Oh, Joonhong Park, Gimoon Hong, Dongyoon Ka, Kyudong Hwang, Jeongje Park, Kyeongpil Kang, Jungyeon Kim, Junyeol Jeon, Nahsung Kim, Yongkee Kwon, Kornijcuk Vladimir, Woojae Shin, Jongsoon Won, Minkyu Lee, Hyunha Joo, Haerang Choi, Jaewook Lee, Donguc Ko, Younggun Jun, Keewon Cho, Ilwoong Kim, Choungki Song, Chunseok Jeong, Daehan Kwon, Jieun Jang, Il Park, Junhyun Chun, and Joohwan Cho. A 1ynm 1.25V 8Gb, 16Gb\/s\/pin GDDR6-based Accelerator-in-Memory supporting 1TFLOPS MAC Operation and Various Activation Functions for Deep-Learning Applications. In 2022 IEEE International Solid-State Circuits Conference (ISSCC), 2022."},{"key":"e_1_3_2_1_49_1","volume-title":"ISCA","author":"Lee Sukhan","year":"2021","unstructured":"Sukhan Lee, Shin-haeng Kang, Jaehoon Lee, Hyeonsu Kim, Eojin Lee, Seungwoo Seo, Hosang Yoon, Seungwon Lee, Kyounghwan Lim, Hyunsung Shin, Jinhyun Kim, O Seongil, Anand Iyer, David Wang, Kyomin Sohn, and Nam Sung Kim. Hardware Architecture and Software Stack for PIM Based on Commercial DRAM Technology : Industrial Product. In ISCA, 2021."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/LCA.2020.2973991"},{"key":"e_1_3_2_1_51_1","first-page":"31199","article-title":"Pre-trained language models for interactive decision-making","volume":"35","author":"Li Shuang","year":"2022","unstructured":"Shuang Li, Xavier Puig, Chris Paxton, Yilun Du, Clinton Wang, Linxi Fan, Tao Chen, De-An Huang, Ekin Aky\u00fcrek, Anima Anandkumar, et al. Pre-trained language models for interactive decision-making. Advances in Neural Information Processing Systems, 35:31199--31212, 2022.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_52_1","volume-title":"OSDI","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph E Gonzalez, et al. {AlpaServe}: Statistical multiplexing with model parallelism for deep learning serving. In OSDI, 2023."},{"key":"e_1_3_2_1_53_1","volume-title":"ISCA","author":"Liu Haifeng","year":"2023","unstructured":"Haifeng Liu, Long Zheng, Yu Huang, Chaoqiang Liu, Xiangyu Ye, Jingrui Yuan, Xiaofei Liao, Hai Jin, and Jingling Xue. Accelerating Personalized Recommendation with Cross-level Near-Memory Processing. In ISCA, 2023."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2016.7446050"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.14778\/3236187.3236188"},{"key":"e_1_3_2_1_56_1","volume-title":"Spotserve: Serving generative large language models on preemptible instances. arXiv preprint arXiv:2311.15566","author":"Miao Xupeng","year":"2023","unstructured":"Xupeng Miao, Chunan Shi, Jiangfei Duan, Xiaoli Xi, Dahua Lin, Bin Cui, and Zhihao Jia. Spotserve: Serving generative large language models on preemptible instances. arXiv preprint arXiv:2311.15566, 2023."},{"key":"e_1_3_2_1_57_1","volume-title":"Github copilot. https:\/\/github.com\/features\/copilot","year":"2022","unstructured":"Microsoft. Github copilot. https:\/\/github.com\/features\/copilot, 2022."},{"key":"e_1_3_2_1_58_1","unstructured":"Facebook Research Microsoft. Onnx: an open format to represent deep learning models. http:\/\/onnx.ai\/ 2017."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2017.54"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"e_1_3_2_1_61_1","unstructured":"Nvidia. Megatron-lm. https:\/\/github.com\/NVIDIA\/Megatron-LM."},{"key":"e_1_3_2_1_62_1","unstructured":"Nvidia. TensorRT-LLM. https:\/\/github.com\/NVIDIA\/TensorRT-LLM."},{"key":"e_1_3_2_1_63_1","volume-title":"https:\/\/developer.nvidia.com\/triton-inference-server","author":"Triton NVIDIA. NVIDIA","year":"2020","unstructured":"NVIDIA. NVIDIA Triton. https:\/\/developer.nvidia.com\/triton-inference-server, 2020."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2022.3202350"},{"key":"e_1_3_2_1_65_1","volume-title":"High-Performance ML Serving. CoRR, abs\/1712.06139","author":"Olston Christopher","year":"2017","unstructured":"Christopher Olston, Noah Fiedel, Kiril Gorovoy, Jeremiah Harmsen, Li Lao, Fangwei Li, Vinu Rajashekhar, Sukriti Ramesh, and Jordan Soyke. TensorFlow-Serving: Flexible, High-Performance ML Serving. CoRR, abs\/1712.06139, 2017."},{"key":"e_1_3_2_1_66_1","volume-title":"chatgpt. https:\/\/chatgpt.com\/blog\/chatgpt","author":"AI.","year":"2023","unstructured":"OpenAI. chatgpt. https:\/\/chatgpt.com\/blog\/chatgpt, 2023."},{"key":"e_1_3_2_1_67_1","volume-title":"Gpt-4 technical report","author":"AI.","year":"2023","unstructured":"OpenAI. Gpt-4 technical report, 2023."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3466752.3480080"},{"key":"e_1_3_2_1_69_1","volume-title":"MICRO","author":"Park Jongse","year":"2016","unstructured":"Jongse Park, Hardik Sharma, Divya Mahajan, Joon Kyung Kim, Preston Olds, and Hadi Esmaeilzadeh. Scale-out acceleration for machine learning. In MICRO, October 2016."},{"key":"e_1_3_2_1_70_1","volume-title":"HPCA","author":"Park Sang-Soo","year":"2024","unstructured":"Sang-Soo Park, Kyung Soo Kim, Jinin So, Jin Jung, Jonggeon Lee, Kyoungwan Woo, Nayeon Kim, Younghyun Lee, Hyungyo Kim, Yongsuk Kwon, Jinhyun Kim, Jieun Lee, YeonGon Cho, Yongmin Tai, Jeonghyeon Cho, Hoyoung Song, Jung Ho Ahn, and Nam Sung Kim. An LPDDR-based CXL-PNM Platform for TCO-Efficient GPT Inference. In HPCA, 2024."},{"key":"e_1_3_2_1_71_1","volume-title":"PyTorch: An Imperative Style","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas K\u00f6pf, Edward Yang, Zach DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. PyTorch: An Imperative Style, High-Performance Deep Learning Library, 2019."},{"key":"e_1_3_2_1_72_1","volume-title":"Efficiently Scaling Transformer Inference","author":"Pope Reiner","year":"2022","unstructured":"Reiner Pope, Sholto Douglas, Aakanksha Chowdhery, Jacob Devlin, James Bradbury, Anselm Levskaya, Jonathan Heek, Kefan Xiao, Shivani Agrawal, and Jeff Dean. Efficiently Scaling Transformer Inference, 2022."},{"key":"e_1_3_2_1_73_1","volume-title":"Hierarchical Text-Conditional Image Generation with CLIP Latents","author":"Ramesh Aditya","year":"2022","unstructured":"Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, and Mark Chen. Hierarchical Text-Conditional Image Generation with CLIP Latents, 2022."},{"key":"e_1_3_2_1_74_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. Code Llama: Open Foundation Models for Code 2023."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.5555\/3195638.3195659"},{"key":"e_1_3_2_1_76_1","volume-title":"Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca","author":"Taori Rohan","year":"2023","unstructured":"Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca, 2023."},{"key":"e_1_3_2_1_77_1","first-page":"33","article-title":"Efficient algorithms for device placement of dnn graph operators","author":"Tarnawski Jakub M","year":"2020","unstructured":"Jakub M Tarnawski, Amar Phanishayee, Nikhil Devanur, Divya Mahajan, and Fanny Nina Paravecino. Efficient algorithms for device placement of dnn graph operators. Advances in Neural Information Processing Systems, 33, 2020.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_78_1","volume-title":"Introducing MPT-30B: Raising the bar for open-source foundation models","author":"Team ML NLP","year":"2023","unstructured":"MosaicML NLP Team. Introducing MPT-30B: Raising the bar for open-source foundation models, 2023."},{"key":"e_1_3_2_1_79_1","volume-title":"https:\/\/sharegpt.com","author":"Team GPT","year":"2023","unstructured":"ShareGPT Team. ShareGPT. https:\/\/sharegpt.com, 2023."},{"key":"e_1_3_2_1_80_1","volume-title":"Llama: Open and efficient foundation language models","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth\u00e9e Lacroix, Baptiste Rozi\u00e8re, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. Llama: Open and efficient foundation language models, 2023."},{"key":"e_1_3_2_1_81_1","volume-title":"Light-Seq: A High Performance Inference Library for Transformers","author":"Wang Xiaohui","year":"2021","unstructured":"Xiaohui Wang, Ying Xiong, Yang Wei, Mingxuan Wang, and Lei Li. Light-Seq: A High Performance Inference Library for Transformers, 2021."},{"key":"e_1_3_2_1_82_1","volume-title":"ASPLOS","author":"Wang Zhengrong","year":"2023","unstructured":"Zhengrong Wang, Christopher Liu, Aman Arora, Lizy John, and Tony Nowatzki. Infinity Stream: Portable and Programmer-Friendly In-\/Near-Memory Fusion. In ASPLOS, 2023."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA51647.2021.00055"},{"key":"e_1_3_2_1_84_1","volume-title":"OSDI","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. Orca: A Distributed Serving System for Transformer-Based Generative Models. In OSDI, 2022."},{"key":"e_1_3_2_1_85_1","volume-title":"Zixuan Ma, Yufei Xue, Jidong Zhai, Wenguang Chen, Peng Zhang, Yuxiao Dong, and Jie Tang. GLM-130B: An Open Bilingual Pre-trained Model","author":"Zeng Aohan","year":"2023","unstructured":"Aohan Zeng, Xiao Liu, Zhengxiao Du, Zihan Wang, Hanyu Lai, Ming Ding, Zhuoyi Yang, Yifan Xu, Wendi Zheng, Xiao Xia, Weng Lam Tam, Zixuan Ma, Yufei Xue, Jidong Zhai, Wenguang Chen, Peng Zhang, Yuxiao Dong, and Jie Tang. GLM-130B: An Open Bilingual Pre-trained Model, 2023."},{"key":"e_1_3_2_1_86_1","volume-title":"Todor Mihaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer. OPT: Open Pre-trained Transformer Language Models","author":"Zhang Susan","year":"2022","unstructured":"Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen, Christopher Dewan, Mona Diab, Xian Li, Xi Victoria Lin, Todor Mihaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer. OPT: Open Pre-trained Transformer Language Models, 2022."},{"key":"e_1_3_2_1_87_1","volume-title":"OSDI","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric P Xing, Joseph Gonzalez, and Ion Stoica. Alpa: Automating inter-and {Intra-Operator} parallelism for distributed deep learning. In OSDI, 2022."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1109\/PACT52795.2021.00021"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA53966.2022.00082"}],"event":{"name":"ASPLOS '24: 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3","location":"La Jolla CA USA","acronym":"ASPLOS '24","sponsor":["SIGARCH ACM Special Interest Group on Computer Architecture","SIGOPS ACM Special Interest Group on Operating Systems","SIGPLAN ACM Special Interest Group on Programming Languages","SIGBED ACM Special Interest Group on Embedded Systems"]},"container-title":["Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3620666.3651380","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:03:43Z","timestamp":1750291423000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3620666.3651380"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,27]]},"references-count":89,"alternative-id":["10.1145\/3620666.3651380","10.1145\/3620666"],"URL":"https:\/\/doi.org\/10.1145\/3620666.3651380","relation":{},"subject":[],"published":{"date-parts":[[2024,4,27]]},"assertion":[{"value":"2024-04-27","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}