{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,7]],"date-time":"2026-02-07T12:22:04Z","timestamp":1770466924829,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,12,11]],"date-time":"2023-12-11T00:00:00Z","timestamp":1702252800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,12,11]]},"DOI":"10.1145\/3631311.3632401","type":"proceedings-article","created":{"date-parts":[[2024,1,8]],"date-time":"2024-01-08T06:15:59Z","timestamp":1704694559000},"page":"13-18","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Towards Optimal Preemptive GPU Time-Sharing for Edge Model Serving"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8626-192X","authenticated-orcid":false,"given":"Zhengxu","family":"Xia","sequence":"first","affiliation":[{"name":"University of Chicago, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4330-1228","authenticated-orcid":false,"given":"Yitian","family":"Hao","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5043-2421","authenticated-orcid":false,"given":"Jun","family":"Duan","sequence":"additional","affiliation":[{"name":"IBM Thomas J. Watson Research Center, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0204-2362","authenticated-orcid":false,"given":"Chen","family":"Wang","sequence":"additional","affiliation":[{"name":"IBM Thomas J. Watson Research Center, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6877-1683","authenticated-orcid":false,"given":"Junchen","family":"Jiang","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,1,8]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Amazon sagemaker. https:\/\/aws.amazon.com\/sagemaker\/. Accessed: 2023-07-07."},{"key":"e_1_3_2_1_2_1","unstructured":"Host multiple models in one container behind one endpoint. https:\/\/docs.aws.amazon.com\/sagemaker\/latest\/dg\/multi-model-endpoints.html. Accessed: 2023-07-07."},{"key":"e_1_3_2_1_3_1","unstructured":"Introducing the hugging face llm inference container for amazon sagemaker. https:\/\/huggingface.co\/blog\/sagemaker-huggingface-llm. Accessed: 2023-07-07."},{"key":"e_1_3_2_1_4_1","unstructured":"Kserve documentation website. https:\/\/kserve.github.io\/website\/0.10\/. Accessed: 2023-07-07."},{"key":"e_1_3_2_1_5_1","unstructured":"Modelmesh. https:\/\/github.com\/kserve\/modelmesh. Accessed: 2023-07-07."},{"key":"e_1_3_2_1_6_1","unstructured":"airplane. Kubernets sidecar container overview. https:\/\/www.airplane.dev\/blog\/kubernetes-sidecar-container."},{"key":"e_1_3_2_1_7_1","unstructured":"Zhihao Bai Zhen Zhang Yibo Zhu and Xin Jin. Pipeswitch source code. https:\/\/github.com\/netx-repo\/PipeSwitch\/tree\/main. Accessed: 2023-06-08."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 14th USENIX Conference on Operating Systems Design and Implementation, OSDI'20, USA","author":"Bai Zhihao","year":"2020","unstructured":"Zhihao Bai, Zhen Zhang, Yibo Zhu, and Xin Jin. Pipeswitch: Fast pipelined context switching for deep learning applications. In Proceedings of the 14th USENIX Conference on Operating Systems Design and Implementation, OSDI'20, USA, 2020. USENIX Association."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3018743.3018748"},{"key":"e_1_3_2_1_10_1","article-title":"Enabling multi-tenant gpu sharing based on kernel burst estimation","author":"Chen Hung-Hsin","year":"2021","unstructured":"Hung-Hsin Chen, En-Te Lin, Yu-Min Chou, and Jerry Chou. Gemini: Enabling multi-tenant gpu sharing based on kernel burst estimation. IEEE Transactions on Cloud Computing, pages 1--1, 2021.","journal-title":"IEEE Transactions on Cloud Computing, pages 1--1"},{"key":"e_1_3_2_1_11_1","volume-title":"Multi-model machine learning inference serving with gpu spatial partitioning. arXiv preprint arXiv:2109.01611","author":"Choi Seungbeom","year":"2021","unstructured":"Seungbeom Choi, Sunho Lee, Yeonjae Kim, Jongse Park, Youngjin Kwon, and Jaehyuk Huh. Multi-model machine learning inference serving with gpu spatial partitioning. arXiv preprint arXiv:2109.01611, 2021."},{"key":"e_1_3_2_1_12_1","first-page":"613","volume-title":"14th USENIX Symposium on Networked Systems Design and Implementation (NSDI 17)","author":"Crankshaw Daniel","year":"2017","unstructured":"Daniel Crankshaw, Xin Wang, Guilio Zhou, Michael J Franklin, Joseph E Gonzalez, and Ion Stoica. Clipper: A {Low-Latency} online prediction serving system. In 14th USENIX Symposium on Networked Systems Design and Implementation (NSDI 17), pages 613--627, 2017."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476143"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/CloudNet51028.2020.9335804"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421284"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPSW.2016.94"},{"key":"e_1_3_2_1_17_1","first-page":"469","volume-title":"Ubiquitous Computing & Communications, Big Data & Cloud Computing, Social Computing & Networking, Sustainable Computing & Communications (ISPA\/IUCC\/BDCloud\/SocialCom\/SustainCom)","author":"Gu Jing","year":"2018","unstructured":"Jing Gu, Shengbo Song, Ying Li, and Hanmei Luo. Gaiagpu: Sharing gpus in container clouds. In 2018 IEEE Intl Conf on Parallel & Distributed Processing with Applications, Ubiquitous Computing & Communications, Big Data & Cloud Computing, Social Computing & Networking, Sustainable Computing & Communications (ISPA\/IUCC\/BDCloud\/SocialCom\/SustainCom), pages 469--476. IEEE, 2018."},{"key":"e_1_3_2_1_18_1","volume-title":"OSDI 20","author":"Gujarati Arpan","year":"2020","unstructured":"Arpan Gujarati, Reza Karimi, Safya Alzayat, Wei Hao, Antoine Kaufmann, Ymir Vigfusson, and Jonathan Mace. Serving DNNs like clockwork: Performance predictability from the bottom up. In OSDI 20, 2020."},{"key":"e_1_3_2_1_19_1","first-page":"539","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Han Mingcong","year":"2022","unstructured":"Mingcong Han, Hanze Zhang, Rong Chen, and Haibo Chen. Microsecond-scale preemption for concurrent {GPU-accelerated} {DNN} inferences. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22), pages 539--558, 2022."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_21_1","unstructured":"Istio. Istio. https:\/\/istio.io\/."},{"key":"e_1_3_2_1_22_1","volume-title":"et al. Alpaserve: Statistical multiplexing with model parallelism for deep learning serving. arXiv preprint arXiv:2302.11665","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph E Gonzalez, et al. Alpaserve: Statistical multiplexing with model parallelism for deep learning serving. arXiv preprint arXiv:2302.11665, 2023."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3582080"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.324"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/RTSS55097.2022.00032"},{"key":"e_1_3_2_1_26_1","unstructured":"NVIDIA. Nvidia multi-instance gpu user guide. https:\/\/docs.nvidia.com\/cuda\/cuda-c-programming-guide\/index.html. Accessed: 2023-07-04."},{"key":"e_1_3_2_1_27_1","first-page":"397","volume-title":"2021 USENIX Annual Technical Conference (USENIX ATC 21)","author":"Romero Francisco","year":"2021","unstructured":"Francisco Romero, Qian Li, Neeraja J Yadwadkar, and Christos Kozyrakis. {INFaaS}: Automated model-less inference serving. In 2021 USENIX Annual Technical Conference (USENIX ATC 21), pages 397--411, 2021."},{"key":"e_1_3_2_1_28_1","volume-title":"ASPLOS 2017","author":"Wu Bo","year":"2017","unstructured":"Bo Wu, Xu Liu, Xiaobo Zhou, and Changjun Jiang. Flep: Enabling flexible and efficient preemption on gpus. In ASPLOS 2017, 2017."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3369583.3392679"},{"key":"e_1_3_2_1_30_1","first-page":"787","volume-title":"20th USENIX Symposium on Networked Systems Design and Implementation (NSDI 23)","author":"Zhang Hong","year":"2023","unstructured":"Hong Zhang, Yupeng Tang, Anurag Khandelwal, and Ion Stoica. {SHEPHERD}: Serving {DNNs} in the wild. In 20th USENIX Symposium on Networked Systems Design and Implementation (NSDI 23), pages 787--808, 2023."},{"key":"e_1_3_2_1_31_1","volume-title":"Gpu-enabled function-as-a-service for machine learning inference. arXiv preprint arXiv:2303.05601","author":"Zhao Ming","year":"2023","unstructured":"Ming Zhao, Kritshekhar Jha, and Sungho Hong. Gpu-enabled function-as-a-service for machine learning inference. arXiv preprint arXiv:2303.05601, 2023."}],"event":{"name":"Middleware '23: 24th International Middleware Conference","location":"Bologna Italy","acronym":"Middleware '23","sponsor":["ACM Association for Computing Machinery","IFIP International Federation for Information Processing"]},"container-title":["Proceedings of the 9th International Workshop on Container Technologies and Container Clouds"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631311.3632401","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3631311.3632401","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T20:36:46Z","timestamp":1755981406000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631311.3632401"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,11]]},"references-count":31,"alternative-id":["10.1145\/3631311.3632401","10.1145\/3631311"],"URL":"https:\/\/doi.org\/10.1145\/3631311.3632401","relation":{},"subject":[],"published":{"date-parts":[[2023,12,11]]},"assertion":[{"value":"2024-01-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}