{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,15]],"date-time":"2025-08-15T00:46:14Z","timestamp":1755218774179,"version":"3.43.0"},"reference-count":58,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100004735","name":"Hunan Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["2023JJ40679"],"award-info":[{"award-number":["2023JJ40679"]}],"id":[{"id":"10.13039\/501100004735","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Neural Networks"],"published-print":{"date-parts":[[2025,10]]},"DOI":"10.1016\/j.neunet.2025.107703","type":"journal-article","created":{"date-parts":[[2025,6,15]],"date-time":"2025-06-15T01:30:36Z","timestamp":1749951036000},"page":"107703","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["ESCAN: Efficient GPU sharing for cascade neural network inference"],"prefix":"10.1016","volume":"190","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0679-4801","authenticated-orcid":false,"given":"Jianan","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Shi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhaoyun","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mei","family":"Wen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"key":"10.1016\/j.neunet.2025.107703_b1","series-title":"International conference on machine learning","first-page":"527","article-title":"Adaptive neural networks for efficient inference","author":"Bolukbasi","year":"2017"},{"issue":"6","key":"10.1016\/j.neunet.2025.107703_b2","first-page":"1319","article-title":"RIFLING: A reinforcement learning-based GPU scheduler for deep learning research and development platforms","volume":"52","author":"Chen","year":"2022","journal-title":"Software: Practice and Experience"},{"key":"10.1016\/j.neunet.2025.107703_b3","unstructured":"Chen, T., Moreau, T., Jiang, Z., Zheng, L., Yan, E., Shen, H., et al. (2018). {TVM}: An automated {End-to-End} optimizing compiler for deep learning. In 13th USeNIX symposium on operating systems design and implementation (OSDI 18) (pp. 578\u2013594)."},{"key":"10.1016\/j.neunet.2025.107703_b4","series-title":"ATC","first-page":"199","article-title":"Serving heterogeneous machine learning models on multi-GPU servers with spatio-temporal sharing","author":"Choi","year":"2022"},{"key":"10.1016\/j.neunet.2025.107703_b5","doi-asserted-by":"crossref","unstructured":"Dhakal, A., et al. GSLICE: controlled spatial sharing of GPUs for a scalable inference platform, SoCC \u201920, 492\u2013506.","DOI":"10.1145\/3419111.3421284"},{"key":"10.1016\/j.neunet.2025.107703_b6","series-title":"9th international conference on learning representations, ICLR 2021, virtual event, Austria, May 3-7, 2021","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","author":"Dosovitskiy","year":"2021"},{"year":"2022","series-title":"Deep learning workload scheduling in GPU datacenters: Taxonomy, challenges and vision","author":"Gao","key":"10.1016\/j.neunet.2025.107703_b7"},{"year":"2017","series-title":"Energy-efficient amortized inference with cascaded deep classifiers","author":"Guan","key":"10.1016\/j.neunet.2025.107703_b8"},{"key":"10.1016\/j.neunet.2025.107703_b9","article-title":"Dynamic neural network structure: A review for its theories and applications","author":"Guo","year":"2024","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"key":"10.1016\/j.neunet.2025.107703_b10","series-title":"International conference on machine learning","first-page":"1321","article-title":"On calibration of modern neural networks","author":"Guo","year":"2017"},{"issue":"11","key":"10.1016\/j.neunet.2025.107703_b11","doi-asserted-by":"crossref","first-page":"7436","DOI":"10.1109\/TPAMI.2021.3117837","article-title":"Dynamic neural networks: A survey","volume":"44","author":"Han","year":"2021","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"10.1016\/j.neunet.2025.107703_b12","doi-asserted-by":"crossref","unstructured":"Harley, A. W., Derpanis, K. G., & Kokkinos, I. (2017). Segmentation-aware convolutional networks using local attention masks. In Proceedings of the IEEE international conference on computer vision (pp. 5038\u20135047).","DOI":"10.1109\/ICCV.2017.539"},{"key":"10.1016\/j.neunet.2025.107703_b13","series-title":"2016 IEEE conference on computer vision and pattern recognition, CVPR 2016, las vegas, NV, USA, June 27-30, 2016","first-page":"770","article-title":"Deep residual learning for image recognition","author":"He","year":"2016"},{"year":"2017","series-title":"Multi-scale dense networks for resource efficient image classification","author":"Huang","key":"10.1016\/j.neunet.2025.107703_b14"},{"key":"10.1016\/j.neunet.2025.107703_b15","series-title":"2017 IEEE conference on computer vision and pattern recognition, CVPR 2017, honolulu, HI, USA, July 21-26, 2017","first-page":"2261","article-title":"Densely connected convolutional networks","author":"Huang","year":"2017"},{"key":"10.1016\/j.neunet.2025.107703_b16","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, S., Van der Maaten, L., & Weinberger, K. Q. (2018). Condensenet: An efficient densenet using learned group convolutions. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 2752\u20132761).","DOI":"10.1109\/CVPR.2018.00291"},{"key":"10.1016\/j.neunet.2025.107703_b17","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, Z., Van Der Maaten, L., & Weinberger, K. Q. (2017c). Densely connected convolutional networks. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 4700\u20134708).","DOI":"10.1109\/CVPR.2017.243"},{"year":"2020","series-title":"Scaling laws for neural language models","author":"Kaplan","key":"10.1016\/j.neunet.2025.107703_b18"},{"key":"10.1016\/j.neunet.2025.107703_b19","series-title":"International conference on machine learning","first-page":"3301","article-title":"Shallow-deep networks: Understanding and mitigating network overthinking","author":"Kaya","year":"2019"},{"key":"10.1016\/j.neunet.2025.107703_b20","series-title":"DAC","first-page":"607","article-title":"PARIS and ELSA: an elastic scheduling algorithm for reconfigurable multi-GPU inference servers","author":"Kim","year":"2022"},{"key":"10.1016\/j.neunet.2025.107703_b21","doi-asserted-by":"crossref","unstructured":"Kontschieder, P., Fiterau, M., Criminisi, A., & Bulo, S. R. (2015). Deep neural decision forests. In Proceedings of the IEEE international conference on computer vision (pp. 1467\u20131475).","DOI":"10.1109\/ICCV.2015.172"},{"year":"2020","series-title":"Cascadebert: Accelerating inference of pre-trained language models via calibrated complete models cascade","author":"Li","key":"10.1016\/j.neunet.2025.107703_b22"},{"key":"10.1016\/j.neunet.2025.107703_b23","series-title":"2023 60th ACM\/IEEE design automation conference","first-page":"1","article-title":"Input-aware dynamic timestep spiking neural networks for efficient in-memory computing","author":"Li","year":"2023"},{"key":"10.1016\/j.neunet.2025.107703_b24","series-title":"AAAI","first-page":"8711","article-title":"Towards inference efficient deep ensemble learning","author":"Li","year":"2023"},{"key":"10.1016\/j.neunet.2025.107703_b25","doi-asserted-by":"crossref","unstructured":"Liu, L., & Deng, J. (2018). Dynamic deep neural networks: Optimizing accuracy-efficiency trade-offs by selective execution. vol. 32, In Proceedings of the AAAI conference on artificial intelligence.","DOI":"10.1609\/aaai.v32i1.11630"},{"key":"10.1016\/j.neunet.2025.107703_b26","doi-asserted-by":"crossref","unstructured":"Liu, Z., Li, J., Shen, Z., Huang, G., Yan, S., & Zhang, C. (2017). Learning efficient convolutional networks through network slimming. In Proceedings of the IEEE international conference on computer vision (pp. 2736\u20132744).","DOI":"10.1109\/ICCV.2017.298"},{"year":"2020","series-title":"Fastbert: a self-distilling bert with adaptive inference time","author":"Liu","key":"10.1016\/j.neunet.2025.107703_b27"},{"issue":"5","key":"10.1016\/j.neunet.2025.107703_b28","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3527155","article-title":"Split computing and early exiting for deep learning applications: Survey and research challenges","volume":"55","author":"Matsubara","year":"2022","journal-title":"ACM Computing Surveys"},{"key":"10.1016\/j.neunet.2025.107703_b29","series-title":"EuroSys","first-page":"80","article-title":"Interference-aware scheduling for inference serving","author":"Mendoza","year":"2021"},{"key":"10.1016\/j.neunet.2025.107703_b30","unstructured":"NI, . https:\/\/www.ni.com\/docs\/zh-CN\/bundle\/labview-real-time-module\/page\/lvrtconcepts\/smp_api.html\/\/."},{"key":"10.1016\/j.neunet.2025.107703_b31","unstructured":"NVIDIA, . Multi-Process Service https:\/\/docs.nvidia.com\/deploy\/mps\/index.html\/\/."},{"key":"10.1016\/j.neunet.2025.107703_b32","unstructured":"NVIDIA, . CUDA-stream https:\/\/developer.nvidia.com\/blog\/gpu-pro-tip-cuda-7-streams-simplify-concurrency\/\/."},{"key":"10.1016\/j.neunet.2025.107703_b33","unstructured":"NVIDIA, . MIG, https:\/\/www.nvidia.com\/en-us\/technologies\/multi-instance-gpu\/."},{"year":"2017","series-title":"Changing model behavior at test-time using reinforcement learning","author":"Odena","key":"10.1016\/j.neunet.2025.107703_b34"},{"key":"10.1016\/j.neunet.2025.107703_b35","series-title":"CVPR","first-page":"10425","article-title":"Designing network design spaces","author":"Radosavovic","year":"2020"},{"key":"10.1016\/j.neunet.2025.107703_b36","series-title":"ATC","first-page":"397","article-title":"Infaas: Automated model-less inference serving","author":"Romero","year":"2021"},{"key":"10.1016\/j.neunet.2025.107703_b37","doi-asserted-by":"crossref","unstructured":"Rota Bulo, S., & Kontschieder, P. (2014). Neural decision forests for semantic image labelling. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 81\u201388).","DOI":"10.1109\/CVPR.2014.18"},{"key":"10.1016\/j.neunet.2025.107703_b38","first-page":"5047","article-title":"Meta-neighborhoods","volume":"33","author":"Shan","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"year":"2017","series-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","author":"Shazeer","key":"10.1016\/j.neunet.2025.107703_b39"},{"key":"10.1016\/j.neunet.2025.107703_b40","unstructured":"Simonyan, K., & Zisserman, A. (2015). Very Deep Convolutional Networks for Large-Scale Image Recognition. In 3rd international conference on learning representations, ICLR 2015, san diego, CA, USA, May 7-9, 2015, conference track proceedings."},{"key":"10.1016\/j.neunet.2025.107703_b41","series-title":"OpML","first-page":"15","article-title":"Deep learning inference service at microsoft","author":"Soifer","year":"2019"},{"key":"10.1016\/j.neunet.2025.107703_b42","series-title":"International conference on machine learning","first-page":"4752","article-title":"Approximation algorithms for cascading prediction models","author":"Streeter","year":"2018"},{"key":"10.1016\/j.neunet.2025.107703_b43","doi-asserted-by":"crossref","unstructured":"Su, H., Jampani, V., Sun, D., Gallo, O., Learned-Miller, E., & Kautz, J. (2019). Pixel-adaptive convolutional neural networks. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 11166\u201311175).","DOI":"10.1109\/CVPR.2019.01142"},{"key":"10.1016\/j.neunet.2025.107703_b44","series-title":"Proceedings of the 36th international conference on machine learning, ICML 2019, 9-15 June 2019, long beach, california, USA","first-page":"6105","article-title":"EfficientNet: Rethinking model scaling for convolutional neural networks","volume":"vol. 97","author":"Tan","year":"2019"},{"year":"2021","series-title":"Serving DNN models with multi-instance GPUs: A case of the reconfigurable machine scheduling problem","author":"Tan","key":"10.1016\/j.neunet.2025.107703_b45"},{"key":"10.1016\/j.neunet.2025.107703_b46","series-title":"2016 23rd international conference on pattern recognition","first-page":"2464","article-title":"Branchynet: Fast inference via early exiting from deep neural networks","author":"Teerapittayanon","year":"2016"},{"key":"10.1016\/j.neunet.2025.107703_b47","series-title":"Proceedings of the 2001 IEEE computer society conference on computer vision and pattern recognition. CVPR 2001","article-title":"Rapid object detection using a boosted cascade of simple features","volume":"vol. 1","author":"Viola","year":"2001"},{"key":"10.1016\/j.neunet.2025.107703_b48","series-title":"The tenth international conference on learning representations, ICLR 2022, virtual event, April 25-29, 2022","article-title":"Wisdom of committees: An overlooked approach to faster and more accurate models","author":"Wang","year":"2022"},{"key":"10.1016\/j.neunet.2025.107703_b49","doi-asserted-by":"crossref","first-page":"65","DOI":"10.1145\/1498765.1498785","article-title":"Roofline: an insightful visual performance model for multicore architectures","author":"Williams","year":"2009","journal-title":"Communications of the ACM"},{"key":"10.1016\/j.neunet.2025.107703_b50","series-title":"APNet","first-page":"36","article-title":"Irina: Accelerating DNN inference with efficient online scheduling","author":"Wu","year":"2020"},{"year":"2020","series-title":"DeeBERT: Dynamic early exiting for accelerating BERT inference","author":"Xin","key":"10.1016\/j.neunet.2025.107703_b51"},{"year":"2022","series-title":"A survey on dynamic neural networks for natural language processing","author":"Xu","key":"10.1016\/j.neunet.2025.107703_b52"},{"key":"10.1016\/j.neunet.2025.107703_b53","doi-asserted-by":"crossref","unstructured":"Yang, L., Han, Y., Chen, X., Song, S., Dai, J., & Huang, G. (2020). Resolution adaptive networks for efficient inference. In Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (pp. 2369\u20132378).","DOI":"10.1109\/CVPR42600.2020.00244"},{"key":"10.1016\/j.neunet.2025.107703_b54","series-title":"ICCAD","first-page":"1","article-title":"Automated runtime-aware scheduling for multi-tenant DNN inference on GPU","author":"Yu","year":"2021"},{"year":"2022","series-title":"A survey of multi-tenant deep learning inference on GPU","author":"Yu","key":"10.1016\/j.neunet.2025.107703_b55"},{"key":"10.1016\/j.neunet.2025.107703_b56","doi-asserted-by":"crossref","first-page":"520","DOI":"10.1016\/j.future.2017.02.036","article-title":"Locality based warp scheduling in GPGPUs","volume":"82","author":"Zhang","year":"2018","journal-title":"Future Generation Computer Systems"},{"key":"10.1016\/j.neunet.2025.107703_b57","first-page":"18330","article-title":"Bert loses patience: Fast and robust inference with early exit","volume":"33","author":"Zhou","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.neunet.2025.107703_b58","doi-asserted-by":"crossref","first-page":"239","DOI":"10.1016\/S0004-3702(02)00190-X","article-title":"Ensembling neural networks: Many could be better than all","volume":"137","author":"Zhou","year":"2002","journal-title":"Artificial Intelligence"}],"container-title":["Neural Networks"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608025005830?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608025005830?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,8,6]],"date-time":"2025-08-06T11:56:43Z","timestamp":1754481403000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0893608025005830"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10]]},"references-count":58,"alternative-id":["S0893608025005830"],"URL":"https:\/\/doi.org\/10.1016\/j.neunet.2025.107703","relation":{},"ISSN":["0893-6080"],"issn-type":[{"type":"print","value":"0893-6080"}],"subject":[],"published":{"date-parts":[[2025,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"ESCAN: Efficient GPU sharing for cascade neural network inference","name":"articletitle","label":"Article Title"},{"value":"Neural Networks","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.neunet.2025.107703","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 Elsevier Ltd. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"107703"}}