{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T16:19:37Z","timestamp":1783095577918,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":89,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,5,8]],"date-time":"2023-05-08T00:00:00Z","timestamp":1683504000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,5,8]]},"DOI":"10.1145\/3552326.3587438","type":"proceedings-article","created":{"date-parts":[[2023,11,21]],"date-time":"2023-11-21T13:18:51Z","timestamp":1700572731000},"page":"233-248","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":64,"title":["Tabi: An Efficient Multi-Level Inference System for Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6596-2337","authenticated-orcid":false,"given":"Yiding","family":"Wang","sequence":"first","affiliation":[{"name":"Hong Kong University of Science and Technology, Clear Water Bay, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2587-6028","authenticated-orcid":false,"given":"Kai","family":"Chen","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Clear Water Bay, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3133-1430","authenticated-orcid":false,"given":"Haisheng","family":"Tan","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6270-2468","authenticated-orcid":false,"given":"Kun","family":"Guo","sequence":"additional","affiliation":[{"name":"Fuzhou University, Fuzhou, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,5,8]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"GlusterFS. https:\/\/www.gluster.org\/."},{"key":"e_1_3_2_1_2_1","unstructured":"HuggingFace models. https:\/\/huggingface.co\/models."},{"key":"e_1_3_2_1_3_1","unstructured":"HuggingFace Tokenizers. https:\/\/github.com\/huggingface\/tokenizers\/."},{"key":"e_1_3_2_1_4_1","unstructured":"HuggingFace Transformers. https:\/\/github.com\/huggingface\/transformers\/."},{"key":"e_1_3_2_1_5_1","unstructured":"PyTorch. https:\/\/pytorch.org\/."},{"key":"e_1_3_2_1_6_1","unstructured":"Redis. https:\/\/redis.io\/."},{"key":"e_1_3_2_1_7_1","unstructured":"TorchServe. https:\/\/github.com\/pytorch\/serve."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00073"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445922"},{"key":"e_1_3_2_1_10_1","first-page":"527","volume-title":"International Conference on Machine Learning","author":"Bolukbasi Tolga","year":"2017","unstructured":"Tolga Bolukbasi, Joseph Wang, Ofer Dekel, and Venkatesh Saligrama. Adaptive neural networks for efficient inference. In International Conference on Machine Learning, pages 527--536. PMLR, 2017."},{"key":"e_1_3_2_1_11_1","volume-title":"On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258","author":"Bommasani Rishi","year":"2021","unstructured":"Rishi Bommasani, Drew A Hudson, Ehsan Adeli, Russ Altman, Simran Arora, Sydney von Arx, Michael S Bernstein, Jeannette Bohg, Antoine Bosselut, Emma Brunskill, et al. On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258, 2021."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00058655"},{"key":"e_1_3_2_1_13_1","volume-title":"Language models are few-shot learners. arXiv preprint arXiv:2005.14165","author":"Brown Tom B","year":"2020","unstructured":"Tom B Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. Language models are few-shot learners. arXiv preprint arXiv:2005.14165, 2020."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-4828"},{"key":"e_1_3_2_1_15_1","first-page":"613","volume-title":"14th {USENIX} Symposium on Networked Systems Design and Implementation ({NSDI} 17)","author":"Crankshaw Daniel","year":"2017","unstructured":"Daniel Crankshaw, Xin Wang, Guilio Zhou, Michael J Franklin, Joseph E Gonzalez, and Ion Stoica. Clipper: A low-latency online prediction serving system. In 14th {USENIX} Symposium on Networked Systems Design and Implementation ({NSDI} 17), pages 613--627, 2017."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.58"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.21"},{"key":"e_1_3_2_1_18_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805, 2018."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.5555\/648054.743935"},{"key":"e_1_3_2_1_20_1","volume-title":"International Conference on Learning Representations","author":"Dosovitskiy Alexey","year":"2020","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, et al. An image is worth 16x16 words: Transformers for image recognition at scale. In International Conference on Learning Representations, 2020."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387514.3405887"},{"key":"e_1_3_2_1_22_1","volume-title":"Pipelines - Hugging Face","author":"Face Hugging","year":"2021","unstructured":"Hugging Face. Pipelines - Hugging Face, 2021. https:\/\/huggingface.co\/docs\/transformers\/main_classes\/pipelines."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441578"},{"key":"e_1_3_2_1_24_1","volume-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity. arXiv preprint arXiv:2101.03961","author":"Fedus William","year":"2021","unstructured":"William Fedus, Barret Zoph, and Noam Shazeer. Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity. arXiv preprint arXiv:2101.03961, 2021."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_1_26_1","volume-title":"et al. Massive: A 1m-example multilingual natural language understanding dataset with 51 typologically-diverse languages. arXiv preprint arXiv:2204.08582","author":"FitzGerald Jack","year":"2022","unstructured":"Jack FitzGerald, Christopher Hench, Charith Peris, Scott Mackie, Kay Rottmann, Ana Sanchez, Aaron Nash, Liam Urbach, Vishesh Kakarala, Richa Singh, et al. Massive: A 1m-example multilingual natural language understanding dataset with 51 typologically-diverse languages. arXiv preprint arXiv:2204.08582, 2022."},{"key":"e_1_3_2_1_27_1","volume-title":"The lottery ticket hypothesis: Finding sparse, trainable neural networks. arXiv preprint arXiv:1803.03635","author":"Frankle Jonathan","year":"2018","unstructured":"Jonathan Frankle and Michael Carbin. The lottery ticket hypothesis: Finding sparse, trainable neural networks. arXiv preprint arXiv:1803.03635, 2018."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00413"},{"key":"e_1_3_2_1_29_1","first-page":"3690","volume-title":"International Conference on Machine Learning","author":"Goyal Saurabh","year":"2020","unstructured":"Saurabh Goyal, Anamitra Roy Choudhury, Saurabh Raje, Venkatesan Chakaravarthy, Yogish Sabharwal, and Ashish Verma. Power-bert: Accelerating bert inference via progressive word-vector elimination. In International Conference on Machine Learning, pages 3690--3699. PMLR, 2020."},{"key":"e_1_3_2_1_30_1","first-page":"443","volume-title":"14th {USENIX} Symposium on Operating Systems Design and Implementation ({OSDI} 20)","author":"Gujarati Arpan","year":"2020","unstructured":"Arpan Gujarati, Reza Karimi, Safya Alzayat, Wei Hao, Antoine Kaufmann, Ymir Vigfusson, and Jonathan Mace. Serving dnns like clockwork: Performance predictability from the bottom up. In 14th {USENIX} Symposium on Operating Systems Design and Implementation ({OSDI} 20), pages 443--462, 2020."},{"key":"e_1_3_2_1_31_1","volume-title":"19th USENIX Symposium on Networked Systems Design and Implementation (NSDI 22)","author":"Gunasekaran Jashwant Raj","year":"2022","unstructured":"Jashwant Raj Gunasekaran, Cyan Subhra Mishra, Prashanth Thinakaran, Mahmut Taylan Kandemir, and Chita R Das. Cocktail: A multidimensional optimization for model serving in cloud. In 19th USENIX Symposium on Networked Systems Design and Implementation (NSDI 22), April 2022."},{"key":"e_1_3_2_1_32_1","first-page":"1321","volume-title":"International Conference on Machine Learning","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian Q Weinberger. On calibration of modern neural networks. In International Conference on Machine Learning, pages 1321--1330. PMLR, 2017."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3514221.3526173"},{"key":"e_1_3_2_1_34_1","volume-title":"The Gradient","author":"Gupta Abhishek","year":"2021","unstructured":"Abhishek Gupta. The imperative for sustainable ai systems. The Gradient, 2021."},{"key":"e_1_3_2_1_35_1","volume-title":"Deberta: Decoding-enhanced bert with disentangled attention. arXiv preprint arXiv:2006.03654","author":"He Pengcheng","year":"2020","unstructured":"Pengcheng He, Xiaodong Liu, Jianfeng Gao, and Weizhu Chen. Deberta: Decoding-enhanced bert with disentangled attention. arXiv preprint arXiv:2006.03654, 2020."},{"key":"e_1_3_2_1_36_1","volume-title":"Clinicalbert: Modeling clinical notes and predicting hospital readmission. arXiv preprint arXiv:1904.05342","author":"Huang Kexin","year":"2019","unstructured":"Kexin Huang, Jaan Altosaar, and Rajesh Ranganath. Clinicalbert: Modeling clinical notes and predicting hospital readmission. arXiv preprint arXiv:1904.05342, 2019."},{"key":"e_1_3_2_1_37_1","unstructured":"Angela H Jiang Daniel L-K Wong Giulio Zhou David G Andersen Jeffrey Dean Gregory R Ganger Gauri Joshi Michael Kaminksy Michael Kozuch Zachary C Lipton et al. Accelerating deep learning by focusing on the biggest losers. arXiv preprint arXiv:1910.00762 2019."},{"key":"e_1_3_2_1_38_1","volume-title":"International Conference on Learning Representations","author":"Kitaev Nikita","year":"2019","unstructured":"Nikita Kitaev, Lukasz Kaiser, and Anselm Levskaya. Reformer: The efficient transformer. In International Conference on Learning Representations, 2019."},{"key":"e_1_3_2_1_39_1","volume-title":"Spvit: Enabling faster vision transformers via soft token pruning. arXiv preprint arXiv:2112.13890","author":"Kong Zhenglun","year":"2021","unstructured":"Zhenglun Kong, Peiyan Dong, Xiaolong Ma, Xin Meng, Wei Niu, Mengshu Sun, Bin Ren, Minghai Qin, Hao Tang, and Yanzhi Wang. Spvit: Enabling faster vision transformers via soft token pruning. arXiv preprint arXiv:2112.13890, 2021."},{"key":"e_1_3_2_1_40_1","volume-title":"Neural network ensembles, cross validation, and active learning. Advances in neural information processing systems, 7:231--238","author":"Krogh Anders","year":"1995","unstructured":"Anders Krogh, Jesper Vedelsby, et al. Neural network ensembles, cross validation, and active learning. Advances in neural information processing systems, 7:231--238, 1995."},{"key":"e_1_3_2_1_41_1","first-page":"2796","volume-title":"International Conference on Machine Learning","author":"Kuleshov Volodymyr","year":"2018","unstructured":"Volodymyr Kuleshov, Nathan Fenner, and Stefano Ermon. Accurate uncertainties for deep learning using calibrated regression. In International Conference on Machine Learning, pages 2796--2804, 2018."},{"key":"e_1_3_2_1_42_1","first-page":"12316","article-title":"Beyond temperature scaling: Obtaining well-calibrated multi-class probabilities with dirichlet calibration","volume":"32","author":"Kull Meelis","year":"2019","unstructured":"Meelis Kull, Miquel Perello Nieto, Markus K\u00e4ngsepp, Telmo Silva Filho, Hao Song, and Peter Flach. Beyond temperature scaling: Obtaining well-calibrated multi-class probabilities with dirichlet calibration. Advances in Neural Information Processing Systems, 32:12316--12326, 2019.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_43_1","volume-title":"Albert: A lite bert for self-supervised learning of language representations. arXiv preprint arXiv:1909.11942","author":"Lan Zhenzhong","year":"2019","unstructured":"Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, and Radu Soricut. Albert: A lite bert for self-supervised learning of language representations. arXiv preprint arXiv:1909.11942, 2019."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1131"},{"key":"e_1_3_2_1_45_1","first-page":"611","volume-title":"13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18)","author":"Lee Yunseong","year":"2018","unstructured":"Yunseong Lee, Alberto Scolari, Byung-Gon Chun, Marco Domenico Santambrogio, Markus Weimer, and Matteo Interlandi. PRETZEL: Opening the black box of machine learning prediction serving systems. In 13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18), pages 611--626. USENIX Association, 2018."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.684"},{"key":"e_1_3_2_1_47_1","volume-title":"Roberta: A robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692","author":"Liu Yinhan","year":"2019","unstructured":"Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. Roberta: A robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692, 2019."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.5555\/2002472.2002491"},{"key":"e_1_3_2_1_49_1","first-page":"546","article-title":"An empirical evaluation of bagging and boosting","volume":"1997","author":"Maclin Richard","year":"1997","unstructured":"Richard Maclin and David Opitz. An empirical evaluation of bagging and boosting. AAAI\/IAAI, 1997:546--551, 1997.","journal-title":"AAAI\/IAAI"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-021-93783-8"},{"key":"e_1_3_2_1_51_1","first-page":"14014","article-title":"Are sixteen heads really better than one?","volume":"32","author":"Michel Paul","year":"2019","unstructured":"Paul Michel, Omer Levy, and Graham Neubig. Are sixteen heads really better than one? Advances in Neural Information Processing Systems, 32:14014--14024, 2019.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476311.3476402"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-456"},{"key":"e_1_3_2_1_54_1","volume-title":"100,000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250","author":"Rajpurkar Pranav","year":"2016","unstructured":"Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, and Percy Liang. Squad: 100,000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250, 2016."},{"key":"e_1_3_2_1_55_1","volume-title":"Dynamicvit: Efficient vision transformers with dynamic token sparsification. Advances in neural information processing systems, 34:13937--13949","author":"Rao Yongming","year":"2021","unstructured":"Yongming Rao, Wenliang Zhao, Benlin Liu, Jiwen Lu, Jie Zhou, and Cho-Jui Hsieh. Dynamicvit: Efficient vision transformers with dynamic token sparsification. Advances in neural information processing systems, 34:13937--13949, 2021."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00349"},{"key":"e_1_3_2_1_57_1","volume-title":"2021 USENIX Annual Technical Conference (USENIX ATC 21)","author":"Romero Francisco","year":"2021","unstructured":"Francisco Romero, Qian Li, Neeraja J Yadwadkar, and Christos Kozyrakis. INFaaS: Automated model-less inference serving. In 2021 USENIX Annual Technical Conference (USENIX ATC 21), July 2021."},{"key":"e_1_3_2_1_58_1","volume-title":"To apply machine learning responsibly, we use it in moderation - New York Times Open","author":"Salganik Matthew J.","year":"2020","unstructured":"Matthew J. Salganik and Robin C. Lee. To apply machine learning responsibly, we use it in moderation - New York Times Open, 2020. https:\/\/open.nytimes.com\/to-apply-machine-learning-responsibly-we-use-it-in-moderation-d001f49e0644."},{"key":"e_1_3_2_1_59_1","volume-title":"a distilled version of bert: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108","author":"Sanh Victor","year":"2019","unstructured":"Victor Sanh, Lysandre Debut, Julien Chaumond, and Thomas Wolf. Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108, 2019."},{"key":"e_1_3_2_1_60_1","volume-title":"Movement pruning: Adaptive sparsity by fine-tuning","author":"Sanh Victor","year":"2020","unstructured":"Victor Sanh, Thomas Wolf, and Alexander M. Rush. Movement pruning: Adaptive sparsity by fine-tuning. 2020."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3381831"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359658"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.89"},{"key":"e_1_3_2_1_64_1","volume-title":"Optimizing prediction serving on low-latency serverless dataflow. arXiv preprint arXiv:2007.05832","author":"Sreekanti Vikram","year":"2020","unstructured":"Vikram Sreekanti, Harikaran Subbaraj, Chenggang Wu, Joseph E Gonzalez, and Joseph M Hellerstein. Optimizing prediction serving on low-latency serverless dataflow. arXiv preprint arXiv:2007.05832, 2020."},{"key":"e_1_3_2_1_65_1","volume-title":"Energy and policy considerations for deep learning in nlp. arXiv preprint arXiv:1906.02243","author":"Strubell Emma","year":"2019","unstructured":"Emma Strubell, Ananya Ganesh, and Andrew McCallum. Energy and policy considerations for deep learning in nlp. arXiv preprint arXiv:1906.02243, 2019."},{"key":"e_1_3_2_1_66_1","volume-title":"Adaptive attention span in transformers. arXiv preprint arXiv:1905.07799","author":"Sukhbaatar Sainbayar","year":"2019","unstructured":"Sainbayar Sukhbaatar, Edouard Grave, Piotr Bojanowski, and Armand Joulin. Adaptive attention span in transformers. arXiv preprint arXiv:1905.07799, 2019."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1441"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR.2016.7900006"},{"key":"e_1_3_2_1_69_1","first-page":"5998","volume-title":"Advances in neural information processing systems","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In Advances in neural information processing systems, pages 5998--6008, 2017."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1580"},{"key":"e_1_3_2_1_71_1","volume-title":"Speeding up transformer CPU inference in Google Cloud - Twitter","author":"Wakabayashi Mariko","year":"2021","unstructured":"Mariko Wakabayashi. Speeding up transformer CPU inference in Google Cloud - Twitter, 2021. https:\/\/blog.twitter.com\/engineering\/en_us\/topics\/insights\/2021\/speeding-up-transformer-cpu-inference-in-google-cloud."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-5446"},{"key":"e_1_3_2_1_73_1","first-page":"3","article-title":"Convnets decomposition via class parallelism for fast inference on live data","author":"Wang Guanhua","year":"2021","unstructured":"Guanhua Wang, Zhuang Liu, Brandon Hsieh, Siyuan Zhuang, Joseph Gonzalez, Trevor Darrell, and Ion Stoica. sensai: Convnets decomposition via class parallelism for fast inference on live data. Proceedings of Machine Learning and Systems, 3, 2021.","journal-title":"Proceedings of Machine Learning and Systems"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA51647.2021.00018"},{"key":"e_1_3_2_1_75_1","first-page":"5776","article-title":"Deep self-attention distillation for task-agnostic compression of pre-trained transformers","volume":"33","author":"Wang Wenhui","year":"2020","unstructured":"Wenhui Wang, Furu Wei, Li Dong, Hangbo Bao, Nan Yang, and Ming Zhou. Minilm: Deep self-attention distillation for task-agnostic compression of pre-trained transformers. Advances in Neural Information Processing Systems, 33:5776--5788, 2020.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM42981.2021.9488801"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1109\/TCC.2022.3142066"},{"key":"e_1_3_2_1_78_1","volume-title":"HotCloud","author":"Wang Yiding","year":"2019","unstructured":"Yiding Wang, Weiyan Wang, Junxue Zhang, Junchen Jiang, and Kai Chen. Bridging the edge-cloud barrier for real-time advanced vision analytics. In HotCloud, 2019."},{"key":"e_1_3_2_1_79_1","unstructured":"Lilian Weng. Attention? attention! 2018. https:\/\/lilianweng.github.io\/lil-log\/2018\/06\/24\/attention-attention.html."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.204"},{"key":"e_1_3_2_1_81_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","volume":"1","author":"Xu Hu","year":"2019","unstructured":"Hu Xu, Bing Liu, Lei Shu, and S Yu Philip. Bert post-training for review reading comprehension and aspect-based sentiment analysis. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 2324--2335, 2019."},{"key":"e_1_3_2_1_82_1","volume-title":"Tacc: A full-stack cloud computing infrastructure for machine learning tasks. arXiv preprint arXiv:2110.01556","author":"Xu Kaiqiang","year":"2021","unstructured":"Kaiqiang Xu, Xinchen Wan, Hao Wang, Zhenghang Ren, Xudong Liao, Decang Sun, Chaoliang Zeng, and Kai Chen. Tacc: A full-stack cloud computing infrastructure for machine learning tasks. arXiv preprint arXiv:2110.01556, 2021."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3492321.3519577"},{"key":"e_1_3_2_1_84_1","volume-title":"NeurIPS","author":"Zaheer Manzil","year":"2020","unstructured":"Manzil Zaheer, Guru Guruganesh, Kumar Avinava Dubey, Joshua Ainslie, Chris Alberti, Santiago Ontanon, Philip Pham, Anirudh Ravula, Qifan Wang, Li Yang, et al. Big bird: Transformers for longer sequences. In NeurIPS, 2020."},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01179"},{"key":"e_1_3_2_1_86_1","first-page":"1049","volume-title":"Mark: Exploiting cloud services for cost-effective, slo-aware machine learning inference serving. In 2019 {USENIX} Annual Technical Conference ({USENIX}{ATC} 19)","author":"Zhang Chengliang","year":"2019","unstructured":"Chengliang Zhang, Minchen Yu, Wei Wang, and Feng Yan. Mark: Exploiting cloud services for cost-effective, slo-aware machine learning inference serving. In 2019 {USENIX} Annual Technical Conference ({USENIX}{ATC} 19), pages 1049--1062, 2019."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544216.3544229"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458864.3467882"},{"key":"e_1_3_2_1_89_1","first-page":"33","article-title":"Bert loses patience: Fast and robust inference with early exit","author":"Zhou Wangchunshu","year":"2020","unstructured":"Wangchunshu Zhou, Canwen Xu, Tao Ge, Julian McAuley, Ke Xu, and Furu Wei. Bert loses patience: Fast and robust inference with early exit. Advances in Neural Information Processing Systems, 33, 2020.","journal-title":"Advances in Neural Information Processing Systems"}],"event":{"name":"EuroSys '23: Eighteenth European Conference on Computer Systems","location":"Rome Italy","acronym":"EuroSys '23","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems"]},"container-title":["Proceedings of the Eighteenth European Conference on Computer Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3552326.3587438","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3552326.3587438","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:39Z","timestamp":1750178859000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3552326.3587438"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,5,8]]},"references-count":89,"alternative-id":["10.1145\/3552326.3587438","10.1145\/3552326"],"URL":"https:\/\/doi.org\/10.1145\/3552326.3587438","relation":{},"subject":[],"published":{"date-parts":[[2023,5,8]]},"assertion":[{"value":"2023-05-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}