{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T23:14:40Z","timestamp":1776122080610,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,28]],"date-time":"2022-06-28T00:00:00Z","timestamp":1656374400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Zhejiang Lab","award":["2021KC0AB04"],"award-info":[{"award-number":["2021KC0AB04"]}]},{"name":"Program for Guangdong Introducing Innovative and Entrepreneurial Teams","award":["2016ZT06D211"],"award-info":[{"award-number":["2016ZT06D211"]}]},{"name":"Natural Science Foundation of China","award":["U1811461"],"award-info":[{"award-number":["U1811461"]}]},{"name":"National Key R&D Program of China","award":["2021YFB0301300"],"award-info":[{"award-number":["2021YFB0301300"]}]},{"name":"CCF-Baidu Open Fund","award":["CCF-BAIDU OF2021032"],"award-info":[{"award-number":["CCF-BAIDU OF2021032"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,28]]},"DOI":"10.1145\/3524059.3532372","type":"proceedings-article","created":{"date-parts":[[2022,6,16]],"date-time":"2022-06-16T16:13:11Z","timestamp":1655395991000},"page":"1-11","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Handling heavy-tailed input of transformer inference on GPUs"],"prefix":"10.1145","author":[{"given":"Jiangsu","family":"Du","sequence":"first","affiliation":[{"name":"Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiazhi","family":"Jiang","sequence":"additional","affiliation":[{"name":"Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"You","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dan","family":"Huang","sequence":"additional","affiliation":[{"name":"Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yutong","family":"Lu","sequence":"additional","affiliation":[{"name":"Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,6,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","author":"Jacob Devlin","year":"2019","unstructured":"Devlin Jacob , Chang Ming-Wei et al., \"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding \". North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT) , 2019 . Devlin Jacob, Chang Ming-Wei et al., \"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\". North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT), 2019."},{"key":"e_1_3_2_1_2_1","volume-title":"Wu Jeff et al., \"Language Models are Unsupervised Multitask Learners","author":"Alec Radford","year":"2019","unstructured":"Radford Alec , Wu Jeff et al., \"Language Models are Unsupervised Multitask Learners \", 2019 . Radford Alec, Wu Jeff et al., \"Language Models are Unsupervised Multitask Learners\", 2019."},{"key":"e_1_3_2_1_3_1","volume-title":"Attention is all you need.\" Advances in neural information processing systems (NIPS)","author":"Ashish Vaswan","year":"2017","unstructured":"Vaswan Ashish , Noam Shazeer , and \" Attention is all you need.\" Advances in neural information processing systems (NIPS) , 2017 . Vaswan Ashish, Noam Shazeer, and et al. \"Attention is all you need.\" Advances in neural information processing systems (NIPS), 2017."},{"key":"e_1_3_2_1_4_1","volume-title":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Socher Richard","year":"2013","unstructured":"Richard Socher , Alex Perelygin et al., \"Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank \". Conference on Empirical Methods in Natural Language Processing (EMNLP) , 2013 . Richard Socher, Alex Perelygin et al., \"Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank\". Conference on Empirical Methods in Natural Language Processing (EMNLP), 2013."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_6_1","volume-title":"Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","author":"Williams Adina","year":"2018","unstructured":"Adina Williams , Nikita Nangia , and Samuel R . Bowman, \"A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference \". Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT) , 2018 . Adina Williams, Nikita Nangia, and Samuel R. Bowman, \"A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference\". Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT), 2018."},{"key":"e_1_3_2_1_7_1","first-page":"2022","article-title":"Accessed on","volume":"18","author":"Nguyen Phi","year":"2022","unstructured":"Phi Nguyen , Nikhil Kulkarni and Optimize your inference jobs using dynamic batch inference with TorchServe on Amazon SageMaker . 2022 . Accessed on : Apr , 18 , 2022 . [Online]. Avaialble: https:\/\/aws.amazon.com\/cn\/blogs\/machine-learning\/optimize-your-inference-jobs-using-dynamic-batch-inference-withtorchserve-on-amazon-sagemaker\/ Phi Nguyen, Nikhil Kulkarni and et al., Optimize your inference jobs using dynamic batch inference with TorchServe on Amazon SageMaker. 2022. Accessed on: Apr, 18, 2022. [Online]. Avaialble: https:\/\/aws.amazon.com\/cn\/blogs\/machine-learning\/optimize-your-inference-jobs-using-dynamic-batch-inference-withtorchserve-on-amazon-sagemaker\/","journal-title":"Apr"},{"key":"e_1_3_2_1_8_1","first-page":"2022","article-title":"Accessed on","volume":"18","author":"Benesty Micha\u00ebl","year":"2021","unstructured":"Micha\u00ebl Benesty . Hugging Face Transformer Inference Under 1 Millisecond Latency . 2021 . Accessed on : Apr , 18 , 2022 . [Online]. Available: https:\/\/towardsdatascience.com\/hugging-face-transformer-inference-under-1-millisecond-latency-e1be0057a51c Micha\u00ebl Benesty. Hugging Face Transformer Inference Under 1 Millisecond Latency. 2021. Accessed on: Apr, 18, 2022. [Online]. Available: https:\/\/towardsdatascience.com\/hugging-face-transformer-inference-under-1-millisecond-latency-e1be0057a51c","journal-title":"Apr"},{"key":"e_1_3_2_1_9_1","volume-title":"7th International Conference on Learning Representations (ICLR)","author":"Wang Alex","year":"2019","unstructured":"Alex Wang , Amanpreet Singh , and et al., \"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding \". In 7th International Conference on Learning Representations (ICLR) , 2019 . Alex Wang, Amanpreet Singh, and et al., \"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding\". In 7th International Conference on Learning Representations (ICLR), 2019."},{"key":"e_1_3_2_1_10_1","volume-title":"Accessed on","author":"Transformer Effective","year":"2022","unstructured":"Effective Transformer . 2022. Accessed on : Feb. 04, 2022 . [Online]. Available: https:\/\/github.com\/bytedance\/effective_transformer Effective Transformer. 2022. Accessed on: Feb. 04, 2022. [Online]. Available: https:\/\/github.com\/bytedance\/effective_transformer"},{"key":"e_1_3_2_1_11_1","volume-title":"Accessed on","year":"2022","unstructured":"FasterTransformer. 2022. Accessed on : Feb. 04, 2022 . [Online]. Available: https:\/\/github.com\/NVIDIA\/FasterTransformer FasterTransformer. 2022. Accessed on: Feb. 04, 2022. [Online]. Available: https:\/\/github.com\/NVIDIA\/FasterTransformer"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441578"},{"key":"e_1_3_2_1_13_1","volume-title":"Processing of the North American Chapter of the Association of Computational Linguistics (NAACL)","author":"Mikolov Tom\u00e1s","year":"2013","unstructured":"Tom\u00e1s Mikolov , Wen-tau Yih , and et al., \"Linguistic Regularities in Continuous Space Word Representations \". In Processing of the North American Chapter of the Association of Computational Linguistics (NAACL) , 2013 . Tom\u00e1s Mikolov, Wen-tau Yih, and et al., \"Linguistic Regularities in Continuous Space Word Representations\". In Processing of the North American Chapter of the Association of Computational Linguistics (NAACL), 2013."},{"key":"e_1_3_2_1_14_1","volume-title":"GPU Technology Conference","author":"Micikevicius P.","year":"2012","unstructured":"P. Micikevicius , \" GPU Performance Analysis and Optimization\". GPU Technology Conference , 2012 . P. Micikevicius, \"GPU Performance Analysis and Optimization\". GPU Technology Conference, 2012."},{"key":"e_1_3_2_1_15_1","volume-title":"Basic Linear Algebra on NVIDIA GPUs","year":"2021","unstructured":"cuBLAS : Basic Linear Algebra on NVIDIA GPUs . 2021 . Accessed on: Sept. 28, 2021. [Online]. Available: https:\/\/developer.nvidia.com\/cublas cuBLAS: Basic Linear Algebra on NVIDIA GPUs. 2021. Accessed on: Sept. 28, 2021. [Online]. Available: https:\/\/developer.nvidia.com\/cublas"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation","author":"Abadi Mart\u00edn","year":"2016","unstructured":"Mart\u00edn Abadi , Paul Barham , and et al., \"TensorFlow: A System for Large-Scale Machine Learning \". In Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation , 2016 . Mart\u00edn Abadi, Paul Barham, and et al., \"TensorFlow: A System for Large-Scale Machine Learning\". In Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation, 2016."},{"key":"e_1_3_2_1_17_1","volume-title":"Advances in neural information processing systems.","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke , Sam Gross , and , Pytorch: An imperative style, high-performance deep learning library . In Advances in neural information processing systems. 2019 . Adam Paszke, Sam Gross, and et al., Pytorch: An imperative style, high-performance deep learning library. In Advances in neural information processing systems. 2019."},{"key":"e_1_3_2_1_18_1","unstructured":"NVIDIA. 2021. TensorRT. https:\/\/developer.nvidia.com\/tensorrt.  NVIDIA. 2021. TensorRT. https:\/\/developer.nvidia.com\/tensorrt."},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the 13th USENIX Conference on Operating Systems Design and Implementation","author":"Chen Tianqi","year":"2018","unstructured":"Tianqi Chen , Thierry Moreau , and et al., \"TVM: An automated end-to-end optimizing compiler for deep learning \". In Proceedings of the 13th USENIX Conference on Operating Systems Design and Implementation , 2018 . Tianqi Chen, Thierry Moreau, and et al., \"TVM: An automated end-to-end optimizing compiler for deep learning\". In Proceedings of the 13th USENIX Conference on Operating Systems Design and Implementation, 2018."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476138"},{"key":"e_1_3_2_1_21_1","unstructured":"Hamid Tabani Ajay Balasubramaniam and etal \"Improving the Efficiency of Transformers for Resource-Constrained Devices\". Arxiv.  Hamid Tabani Ajay Balasubramaniam and et al. \"Improving the Efficiency of Transformers for Resource-Constrained Devices\". Arxiv."},{"key":"e_1_3_2_1_22_1","volume-title":"Synthesizer: Rethinking Self-Attention in Transformer Models","author":"Tay Yi","year":"2020","unstructured":"Yi Tay , Dara Bahri , and , \" Synthesizer: Rethinking Self-Attention in Transformer Models \", 2020 , Arxiv . Yi Tay, Dara Bahri, and et al., \"Synthesizer: Rethinking Self-Attention in Transformer Models\", 2020, Arxiv."}],"event":{"name":"ICS '22: 2022 International Conference on Supercomputing","location":"Virtual Event","acronym":"ICS '22","sponsor":["SIGARCH ACM Special Interest Group on Computer Architecture"]},"container-title":["Proceedings of the 36th ACM International Conference on Supercomputing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524059.3532372","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3524059.3532372","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:37Z","timestamp":1750188637000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524059.3532372"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,28]]},"references-count":22,"alternative-id":["10.1145\/3524059.3532372","10.1145\/3524059"],"URL":"https:\/\/doi.org\/10.1145\/3524059.3532372","relation":{},"subject":[],"published":{"date-parts":[[2022,6,28]]},"assertion":[{"value":"2022-06-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}