{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T09:04:20Z","timestamp":1743066260746,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031703614"},{"type":"electronic","value":"9783031703621"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-70362-1_5","type":"book-chapter","created":{"date-parts":[[2024,8,29]],"date-time":"2024-08-29T07:03:03Z","timestamp":1724914983000},"page":"73-88","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Input Compression with\u00a0Positional Consistency for\u00a0Efficient Training and\u00a0Inference of\u00a0Transformer Neural Networks"],"prefix":"10.1007","author":[{"given":"Amrit","family":"Nagarajan","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anand","family":"Raghunathan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,8,22]]},"reference":[{"key":"5_CR1","unstructured":"Bird, S.: NLTK: the natural language toolkit. In: ACL 2006, 44th Annual Meeting of the Association for Computational Linguistics, Proceedings of the Conference, Sydney, Australia (2006). https:\/\/aclanthology.org\/P06-4018\/"},{"key":"5_CR2","doi-asserted-by":"publisher","unstructured":"Dehghani, M., et al.: Patch n\u2019 Pack: NaViT, a vision transformer for any aspect ratio and resolution. CoRR (2023). https:\/\/doi.org\/10.48550\/arXiv.2307.06304","DOI":"10.48550\/arXiv.2307.06304"},{"key":"5_CR3","doi-asserted-by":"publisher","unstructured":"Deng, J., Dong, W., Socher, R., Li, L., Li, K., Fei-Fei, L.: ImageNet: a large-scale hierarchical image database. In: 2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR 2009), 20-25 June 2009 (2009). https:\/\/doi.org\/10.1109\/CVPR.2009.5206848","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"5_CR4","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis (2019). https:\/\/doi.org\/10.18653\/v1\/n19-1423","DOI":"10.18653\/v1\/n19-1423"},{"key":"5_CR5","unstructured":"Dosovitskiy, A., et al.: An image is worth 16$$\\times $$16 words: Transformers for image recognition at scale. In: 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria (2021). https:\/\/openreview.net\/forum?id=YicbFdNTTy"},{"key":"5_CR6","doi-asserted-by":"crossref","unstructured":"Gong, Y., Chung, Y., Glass, J.R.: AST: audio spectrogram transformer. CoRR (2021). https:\/\/arxiv.org\/abs\/2104.01778","DOI":"10.21437\/Interspeech.2021-698"},{"key":"5_CR7","doi-asserted-by":"publisher","unstructured":"Goyal, R., et al.: The \u201csomething somethin\u201d video database for learning and evaluating visual common sense. In: IEEE International Conference on Computer Vision, ICCV 2017, Venice, Italy (2017). https:\/\/doi.org\/10.1109\/ICCV.2017.622","DOI":"10.1109\/ICCV.2017.622"},{"key":"5_CR8","doi-asserted-by":"publisher","unstructured":"Han, Y., Huang, G., Song, S., Yang, L., Wang, H., Wang, Y.: Dynamic neural networks: a survey. IEEE Trans. Pattern Anal. Mach. Intell. (2022). https:\/\/doi.org\/10.1109\/TPAMI.2021.3117837","DOI":"10.1109\/TPAMI.2021.3117837"},{"key":"5_CR9","unstructured":"Hayes, P.J., Weinstein, S.P.: CONSTRUE\/TIS: a system for content-based indexing of a database of news stories. In: Proceedings of the The Second Conference on Innovative Applications of Artificial Intelligence (IAAI-90), Washington, DC, USA (1990). http:\/\/www.aaai.org\/Library\/IAAI\/1990\/iaai90-006.php"},{"key":"5_CR10","unstructured":"Hendrycks, D., Mu, N., Cubuk, E.D., Zoph, B., Gilmer, J., Lakshminarayanan, B.: AugMix: a simple data processing method to improve robustness and uncertainty (2020)"},{"key":"5_CR11","unstructured":"Hendrycks, D., Mu, N., Cubuk, E.D., Zoph, B., Gilmer, J., Lakshminarayanan, B.: AugMix: a simple data processing method to improve robustness and uncertainty. In: 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, 26-30 April 2020 (2020). https:\/\/openreview.net\/forum?id=S1gmrxHFvB"},{"key":"5_CR12","unstructured":"Kaplan, J., et al.: Scaling laws for neural language models. CoRR (2020). https:\/\/arxiv.org\/abs\/2001.08361"},{"key":"5_CR13","unstructured":"Kay, W., et al.: The kinetics human action video dataset. CoRR (2017). http:\/\/arxiv.org\/abs\/1705.06950"},{"key":"5_CR14","doi-asserted-by":"crossref","unstructured":"Koutini, K., Schl\u00fcter, J., Eghbal-zadeh, H., Widmer, G.: Efficient training of audio transformers with patchout. CoRR (2021). https:\/\/arxiv.org\/abs\/2110.05069","DOI":"10.21437\/Interspeech.2022-227"},{"key":"5_CR15","volume-title":"Learning Multiple Layers of Features from Tiny Images","author":"A Krizhevsky","year":"2009","unstructured":"Krizhevsky, A.: Learning Multiple Layers of Features from Tiny Images. Univ. Toronto, Technical report (2009)"},{"key":"5_CR16","doi-asserted-by":"crossref","unstructured":"Li, K., et al.: Unmasked teacher: towards training-efficient video foundation models. CoRR (2023). https:\/\/doi.org\/10.48550\/arXiv.2303.16058","DOI":"10.1109\/ICCV51070.2023.01826"},{"key":"5_CR17","unstructured":"Liu, Y., et al.: RoBERTa: a robustly optimized BERT pretraining approach. CoRR (2019). http:\/\/arxiv.org\/abs\/1907.11692"},{"key":"5_CR18","doi-asserted-by":"crossref","unstructured":"Park, D.S., et al: SpecAugment: a simple data augmentation method for automatic speech recognition. In: Interspeech 2019, 20th Annual Conference of the International Speech Communication Association, Graz, Austria (2019). https:\/\/doi.org\/10.21437\/Interspeech.2019-2680","DOI":"10.21437\/Interspeech.2019-2680"},{"key":"5_CR19","doi-asserted-by":"crossref","unstructured":"Piczak, K.J.: ESC: dataset for environmental sound classification. In: Proceedings of the 23rd Annual ACM Conference on Multimedia Conference, MM 2015, Brisbane, Australia (2015). https:\/\/doi.org\/10.1145\/2733373.2806390","DOI":"10.1145\/2733373.2806390"},{"key":"5_CR20","doi-asserted-by":"publisher","unstructured":"Teerapittayanon, S., McDanel, B., Kung, H.T.: BranchyNet: fast inference via early exiting from deep neural networks. In: 23rd International Conference on Pattern Recognition, ICPR 2016, Canc\u00fan, Mexico (2016). https:\/\/doi.org\/10.1109\/ICPR.2016.7900006","DOI":"10.1109\/ICPR.2016.7900006"},{"key":"5_CR21","unstructured":"Verma, V., et al.: Manifold mixup: better representations by interpolating hidden states (2019)"},{"key":"5_CR22","doi-asserted-by":"crossref","unstructured":"Wang, A., Singh, A., Michael, J., Hill, F., Levy, O., Bowman, S.: GLUE: a multi-task benchmark and analysis platform for natural language understanding. In: Proceedings of the 2018 EMNLP Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP (2018). https:\/\/aclanthology.org\/W18-5446","DOI":"10.18653\/v1\/W18-5446"},{"key":"5_CR23","unstructured":"Wang, Y., Huang, R., Song, S., Huang, Z., Huang, G.: Not all images are worth 16$$\\times $$16 words: dynamic transformers for efficient image recognition. In: Advances in Neural Information Processing Systems, vol. 34: NeurIPS 2021 (2021). https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/64517d8435994992e682b3e4aa0a0661-Abstract.html"},{"key":"5_CR24","unstructured":"Warden, P.: Speech commands: a dataset for limited-vocabulary speech recognition. CoRR abs\/1804.03209 (2018). http:\/\/arxiv.org\/abs\/1804.03209"},{"key":"5_CR25","doi-asserted-by":"crossref","unstructured":"Wei, J.W., Zou, K.: EDA: easy data augmentation techniques for boosting performance on text classification tasks. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing, EMNLP-IJCNLP 2019, Hong Kong, China, 3-7 November 2019 (2019)","DOI":"10.18653\/v1\/D19-1670"},{"key":"5_CR26","unstructured":"Wolf, T., et al.: Huggingface\u2019s transformers: state-of-the-art natural language processing. CoRR (2019). http:\/\/arxiv.org\/abs\/1910.03771"},{"key":"5_CR27","unstructured":"Xie, E., Wang, W., Yu, Z., Anandkumar, A., \u00c1lvarez, J.M., Luo, P.: SegFormer: simple and efficient design for semantic segmentation with transformers. In: Advances in Neural Information Processing Systems, vol. 34: NeurIPS 2021 (2021). https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/64f1f27bf1b4ec22924fd0acb550c235-Abstract.html"},{"key":"5_CR28","doi-asserted-by":"publisher","unstructured":"Yun, S., Han, D., Chun, S., Oh, S.J., Yoo, Y., Choe, J.: Cutmix: regularization strategy to train strong classifiers with localizable features. In: 2019 IEEE\/CVF International Conference on Computer Vision, ICCV 2019, Seoul, Korea (South) (2019). https:\/\/doi.org\/10.1109\/ICCV.2019.00612","DOI":"10.1109\/ICCV.2019.00612"},{"key":"5_CR29","unstructured":". Zhang, H., Ciss\u00e9, M., Dauphin, Y.N., Lopez-Paz, D.: mixup: beyond empirical risk minimization. In: 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, Conference Track Proceedings (2018). https:\/\/openreview.net\/forum?id=r1Ddp1-Rb"}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases. Research Track"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-70362-1_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,29]],"date-time":"2024-08-29T07:03:40Z","timestamp":1724915020000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-70362-1_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031703614","9783031703621"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-70362-1_5","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"22 August 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Vilnius","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lithuania","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2024.ecmlpkdd.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}