{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T18:37:31Z","timestamp":1782931051845,"version":"3.54.5"},"reference-count":57,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,8,4]],"date-time":"2025-08-04T00:00:00Z","timestamp":1754265600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,8,4]],"date-time":"2025-08-04T00:00:00Z","timestamp":1754265600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,8,4]]},"DOI":"10.1109\/coins65080.2025.11125797","type":"proceedings-article","created":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T23:57:49Z","timestamp":1755907069000},"page":"1-8","source":"Crossref","is-referenced-by-count":5,"title":["LLM-NPU: Towards Efficient Foundation Model Inference on Low-Power Neural Processing Units"],"prefix":"10.1109","author":[{"given":"Arnab","family":"Raha","sequence":"first","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Souvik","family":"Kundu","sequence":"additional","affiliation":[{"name":"Intel Labs,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sharath Nittur","family":"Sridhar","sequence":"additional","affiliation":[{"name":"Intel Labs,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shamik","family":"Kundu","sequence":"additional","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Soumendu Kumar","family":"Ghosh","sequence":"additional","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alessandro","family":"Palla","sequence":"additional","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arghadip","family":"Das","sequence":"additional","affiliation":[{"name":"Purdue University,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Darren","family":"Crews","sequence":"additional","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Deepak A.","family":"Mathaikutty","sequence":"additional","affiliation":[{"name":"NPU IP, Intel Corporation,USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.3389\/fhpcp.2025.1570210"},{"key":"ref2","article-title":"2024 Intel Tech Tour: LNL AI Hardware Accelerators \u2014 intel.com","year":"2024"},{"key":"ref3","article-title":"The Llama 3 herd of models","author":"Grattafiori","year":"2024"},{"key":"ref4","article-title":"Qwen3 technical report","author":"Yang","year":"2025"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.172"},{"key":"ref6","article-title":"The language model evaluation harness","author":"Gao","year":"2024"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TCSI.2025.3546256"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ISSCC49657.2024.10454509"},{"key":"ref9","article-title":"NITRO: LLM inference on Intel laptop NPUs","author":"Fei","year":"2024"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3669940.3707239"},{"key":"ref11","first-page":"57053","article-title":"Junk DNA hypothesis: Pruning small pre-trained weights irreversibly and monotonically impairs \"difficult\" downstream tasks in llms","volume-title":"Proc. ICML","volume":"235","author":"Yin"},{"key":"ref12","article-title":"Accelerating LLM inference with flexible N: M sparsity via a fully digital compute-in-memory accelerator","author":"Ramachandran","year":"2025"},{"key":"ref13","first-page":"22947","article-title":"SnapKV: LLM knows what you are looking for before generation","volume-title":"Proc. NeurIPS","volume":"37","author":"Li"},{"key":"ref14","first-page":"24822","article-title":"ShiftAddLLM: Accelerating pretrained LLMs via post-training multiplication-less reparameterization","volume-title":"Proc. NeurIPS","volume":"37","author":"You"},{"key":"ref15","first-page":"305","article-title":"GEAR: An efficient error reduction framework for KV cache compression in LLM inference","volume-title":"Proc. NeurIPS","volume":"262","author":"Kang"},{"key":"ref16","first-page":"87","article-title":"AWQ: Activation-aware weight quantization for on-device LLM compression and acceleration","volume-title":"Proc. MLSys","volume":"6","author":"Lin"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/2684746.2689060"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19568-6_5"},{"key":"ref19","first-page":"68658","article-title":"Flashattention-3: Fast and accurate attention with asynchrony and low-precision","volume-title":"Proc. NeurIPS","volume":"37","author":"Shah"},{"key":"ref20","first-page":"16344","article-title":"Flashattention: Fast and memory-efficient exact attention with io-awareness","volume-title":"Proc. NeurIPS","volume":"35","author":"Dao"},{"key":"ref21","article-title":"LLM inference acceleration via efficient operation fusion","author":"Salmani","year":"2025"},{"key":"ref22","article-title":"ONNX runtime","year":"2021"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2024.3423692"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA52012.2021.00011"},{"key":"ref25","first-page":"134","article-title":"7.1 A 3.4-to-13.3 TOPS\/W 3.6 TOPS dual-core deep-learning accelerator for versatile AI applications in 7nm 5G smartphone SoC","volume-title":"Proc. ISSCC","author":"Lin"},{"key":"ref26","article-title":"Fastattention: Extend flashattention2 to NPUs and low-resource GPUs","author":"Lin","year":"2024"},{"key":"ref27","article-title":"MambaExtend: A training-free approach to improve long context extension of Mamba","volume-title":"Proc. ICLR","author":"Azizi"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-short.96"},{"key":"ref29","first-page":"34661","article-title":"H2O: Heavy-hitter oracle for efficient generative inference of large language models","volume-title":"Proc. NeurIPS","volume":"36","author":"Zhang"},{"key":"ref30","first-page":"47901","article-title":"QUEST: Query-aware sparsity for efficient long-context LLM inference","volume-title":"Proc. ICML","volume":"235","author":"Tang"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"ref32","first-page":"19274","article-title":"Fast inference from transformers via speculative decoding","volume-title":"Proc. ICML","volume":"202","author":"Leviathan"},{"key":"ref33","article-title":"LANTERN++: Enhanced relaxed speculative decoding with static tree drafting for visual auto-regressive models","author":"Park","year":"2025","journal-title":"ICLR W"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/IGSC54211.2021.9651614"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA53966.2022.00032"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ISSCC49657.2024.10454395"},{"key":"ref37","article-title":"ASiM: Improving transparency of SRAM-based analog compute-in-memory research with an open-source simulation framework","author":"Zhang","year":"2024"},{"key":"ref38","first-page":"252","article-title":"16.4 An 89TOPS\/W and 16.3 TOPS\/mm 2 all-digital SRAM-based full-precision compute-in memory macro in 22nm for machine-learning edge applications","volume-title":"Proc. ISSCC","volume":"64","author":"Chih"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/jssc.2019.2952773"},{"key":"ref40","first-page":"101969","article-title":"Learn to be efficient: Build structured sparsity in large language models","volume-title":"Proc. NeurIPS","volume":"37","author":"Zheng"},{"key":"ref41","article-title":"Accelerating sparsity in the NVIDIA ampere architecture","author":"Pool","year":"2020","journal-title":"GTC 2020"},{"key":"ref42","article-title":"Optimized spatial architecture mapping flow for transformer accelerators","author":"Xu","year":"2024"},{"key":"ref43","first-page":"414","article-title":"Transcending runtime-memory tradeoffs in checkpointing by being fusion aware","volume-title":"Proc. MLSys","volume":"5","author":"He"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-021-94691-7"},{"key":"ref45","article-title":"Faster lookup table evaluation with application to secure LLM inference","author":"Hou","year":"2024","journal-title":"Cryptology ePrint Archive"},{"key":"ref46","article-title":"Microscaling data formats for deep learning","author":"Rouhani","year":"2023"},{"key":"ref47","first-page":"10088","article-title":"QLoRA: Efficient finetuning of quantized LLMs","volume-title":"Proc. NeurIPS","volume":"36","author":"Dettmers"},{"key":"ref48","first-page":"11573","article-title":"Learning from students: Applying t-distributions to explore accurate and efficient formats for LLMs","volume-title":"Proc. ICML","volume":"235","author":"Dotzel"},{"key":"ref49","article-title":"Qualcomm cloud AI 100 accelerates large language model inference ~2x using microscaling (Mx) formats","year":"2024"},{"key":"ref50","article-title":"GPTQ: Accurate post-training quantization for generative pre-trained transformers","author":"Frantar","year":"2022"},{"key":"ref51","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref52","article-title":"RULER: What\u2019s the real context size of your long-context language models?","author":"Hsieh","year":"2024"},{"key":"ref53","article-title":"Stanford Alpaca: An instruction-following LLaMA model","author":"Taori","year":"2023"},{"key":"ref54","article-title":"Intel\u00ae Core\u2122 Ultra (Series 2) Product Brief","year":"2025"},{"key":"ref55","article-title":"HWiNFO: Freeware system monitoring, profiling, and diagnostics software","year":"2025"},{"key":"ref56","article-title":"VPU-EM: An event-based modeling framework to evaluate NPU performance and power efficiency at scale","author":"Qi","year":"2023"},{"key":"ref57","article-title":"Intel\u00ae Core\u2122 Ultra 7 Processor 268V","year":"2024"}],"event":{"name":"2025 IEEE International Conference on Omni-layer Intelligent Systems (COINS)","location":"Madison, WI, USA","start":{"date-parts":[[2025,8,4]]},"end":{"date-parts":[[2025,8,6]]}},"container-title":["2025 IEEE International Conference on Omni-layer Intelligent Systems (COINS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11125678\/11125719\/11125797.pdf?arnumber=11125797","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,27]],"date-time":"2026-01-27T04:45:12Z","timestamp":1769489112000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11125797\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,4]]},"references-count":57,"URL":"https:\/\/doi.org\/10.1109\/coins65080.2025.11125797","relation":{},"subject":[],"published":{"date-parts":[[2025,8,4]]}}}