{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,8]],"date-time":"2026-05-08T00:26:16Z","timestamp":1778199976703,"version":"3.51.4"},"publisher-location":"Cham","reference-count":65,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031703409","type":"print"},{"value":"9783031703416","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-70341-6_1","type":"book-chapter","created":{"date-parts":[[2024,8,30]],"date-time":"2024-08-30T20:26:39Z","timestamp":1725049599000},"page":"3-20","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Adaptive Sparsity Level During Training for\u00a0Efficient Time Series Forecasting with\u00a0Transformers"],"prefix":"10.1007","author":[{"given":"Zahra","family":"Atashgahi","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mykola","family":"Pechenizkiy","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Raymond","family":"Veldhuis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Decebal Constantin","family":"Mocanu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,8,22]]},"reference":[{"key":"1_CR1","doi-asserted-by":"crossref","unstructured":"Atashgahi, Z., et al.: Quick and robust feature selection: the strength of energy-efficient sparse training for autoencoders. Mach. Learn. 1\u201338 (2022)","DOI":"10.1007\/s10994-021-06063-x"},{"key":"1_CR2","unstructured":"Box, G.E., Jenkins, G.M., Reinsel, G.C., Ljung, G.M.: Time Series Analysis: Forecasting and Control. Wiley, New York (2015)"},{"key":"1_CR3","doi-asserted-by":"crossref","unstructured":"Challu, C., Olivares, K.G., Oreshkin, B.N., Garza, F., Mergenthaler, M., Dubrawski, A.: N-hits: neural hierarchical interpolation for time series forecasting. arXiv preprint arXiv:2201.12886 (2022)","DOI":"10.1609\/aaai.v37i6.25854"},{"key":"1_CR4","first-page":"19974","volume":"34","author":"T Chen","year":"2021","unstructured":"Chen, T., Cheng, Y., Gan, Z., Yuan, L., Zhang, L., Wang, Z.: Chasing sparsity in vision transformers: an end-to-end exploration. Adv. Neural. Inf. Process. Syst. 34, 19974\u201319988 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"1_CR5","unstructured":"Chen, T., et al.: The lottery ticket hypothesis for pre-trained BERT networks. In: Larochelle, H., Ranzato, M., Hadsell, R., Balcan, M., Lin, H. (eds.) Advances in Neural Information Processing Systems, vol.\u00a033, pp. 15834\u201315846 (2020)"},{"key":"1_CR6","doi-asserted-by":"crossref","unstructured":"Curci, S., Mocanu, D.C., Pechenizkiyi, M.: Truly sparse neural networks at scale. arXiv preprint arXiv:2102.01732 (2021)","DOI":"10.21203\/rs.3.rs-133395\/v1"},{"key":"1_CR7","unstructured":"Dietrich, A.S.D., Gressmann, F., Orr, D., Chelombiev, I., Justus, D., Luschi, C.: Towards structured dynamic sparse pre-training of BERT (2022)"},{"key":"1_CR8","unstructured":"Dosovitskiy, A., et al.: An image is worth 16\u00a0$$\\times $$\u00a016 words: transformers for image recognition at scale. In: International Conference on Learning Representations (2021)"},{"key":"1_CR9","unstructured":"Evci, U., Gale, T., Menick, J., Castro, P.S., Elsen, E.: Rigging the lottery: making all tickets winners. In: International Conference on Machine Learning (2020)"},{"key":"1_CR10","unstructured":"Franceschi, J.Y., Dieuleveut, A., Jaggi, M.: Unsupervised scalable representation learning for multivariate time series. In: Advances in Neural Information Processing Systems (2019)"},{"key":"1_CR11","unstructured":"Frankle, J., Carbin, M.: The lottery ticket hypothesis: finding sparse, trainable neural networks. In: International Conference on Learning Representations (2018)"},{"key":"1_CR12","unstructured":"Furuya, T., Suetake, K., Taniguchi, K., Kusumoto, H., Saiin, R., Daimon, T.: Spectral pruning for recurrent neural networks. In: International Conference on Artificial Intelligence and Statistics (2022)"},{"key":"1_CR13","doi-asserted-by":"crossref","unstructured":"Ganesh, P., et al.: Compressing large-scale transformer-based models: a case study on BERT. Trans. Assoc. Comput. Linguist. 9, 1061\u20131080 (2021)","DOI":"10.1162\/tacl_a_00413"},{"key":"1_CR14","unstructured":"Han, S., et al.: DSD: dense-sparse-dense training for deep neural networks. In: International Conference on Learning Representations (2017)"},{"key":"1_CR15","unstructured":"Han, S., Pool, J., Tran, J., Dally, W.: Learning both weights and connections for efficient neural network. In: Advances in Neural Information Processing Systems (2015)"},{"key":"1_CR16","unstructured":"Hoefler, T., Alistarh, D., Ben-Nun, T., Dryden, N., Peste, A.: Sparsity in deep learning: pruning and growth for efficient inference and training in neural networks. J. Mach. Learn. Res. 22(241), 1\u2013124 (2021)"},{"key":"1_CR17","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1016\/j.csda.2015.11.007","volume":"97","author":"RJ Hyndman","year":"2016","unstructured":"Hyndman, R.J., Lee, A.J., Wang, E.: Fast computation of reconciled forecasts for hierarchical and grouped time series. Comput. Stat. Data Anal. 97, 16\u201332 (2016)","journal-title":"Comput. Stat. Data Anal."},{"key":"1_CR18","unstructured":"Jayakumar, S., Pascanu, R., Rae, J., Osindero, S., Elsen, E.: Top-KAST: Top-K always sparse training. In: Advances in Neural Information Processing Systems (2020)"},{"key":"1_CR19","unstructured":"Jin, X., Park, Y., Maddix, D., Wang, H., Wang, Y.: Domain adaptation for time series forecasting via attention sharing. In: International Conference on Machine Learning, pp. 10280\u201310297. PMLR (2022)"},{"key":"1_CR20","doi-asserted-by":"crossref","unstructured":"Kieu, T., Yang, B., Guo, C., Jensen, C.S.: Outlier detection for time series with recurrent autoencoder ensembles. In: IJCAI, pp. 2725\u20132732 (2019)","DOI":"10.24963\/ijcai.2019\/378"},{"key":"1_CR21","unstructured":"Kitaev, N., Kaiser, L., Levskaya, A.: Reformer: the efficient transformer. In: International Conference on Learning Representations (2020)"},{"key":"1_CR22","doi-asserted-by":"crossref","unstructured":"Lai, G., Chang, W.C., Yang, Y., Liu, H.: Modeling long-and short-term temporal patterns with deep neural networks. In: The 41st international ACM SIGIR Conference on Research Development in Information Retrieval, pp. 95\u2013104 (2018)","DOI":"10.1145\/3209978.3210006"},{"key":"1_CR23","unstructured":"Lee, N., Ajanthan, T., Torr, P.: SNIP: single-shot network pruning based on connection sensitivity. In: International Conference on Learning Representations (2019)"},{"key":"1_CR24","unstructured":"Li, S., et al.: Enhancing the locality and breaking the memory bottleneck of transformer on time series forecasting. In: Advances in Neural Information Processing Systems, vol.\u00a032 (2019)"},{"key":"1_CR25","unstructured":"Li, Y., Lu, X., Wang, Y., Dou, D.: Generative time series forecasting with diffusion, denoise, and disentanglement. In: Advances in Neural Information Processing Systems (2022)"},{"key":"1_CR26","unstructured":"Li, Z., et al.: Train big, then compress: rethinking model size for efficient training and inference of transformers. In: International Conference on Machine Learning (2020)"},{"issue":"2194","key":"1_CR27","doi-asserted-by":"publisher","first-page":"20200209","DOI":"10.1098\/rsta.2020.0209","volume":"379","author":"B Lim","year":"2021","unstructured":"Lim, B., Zohren, S.: Time-series forecasting with deep learning: a survey. Phil. Trans. R. Soc. A 379(2194), 20200209 (2021)","journal-title":"Phil. Trans. R. Soc. A"},{"key":"1_CR28","unstructured":"Liu, S., et al.: Sparse training via boosting pruning plasticity with neuroregeneration. Adv. Neural Inf. Process. Syst. 34, 9908\u20139922 (2021)"},{"key":"1_CR29","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/978-3-030-67664-3_17","volume-title":"Machine Learning and Knowledge Discovery in Databases","author":"S Liu","year":"2021","unstructured":"Liu, S., et al.: Topological insights into sparse neural networks. In: Hutter, F., Kersting, K., Lijffijt, J., Valera, I. (eds.) ECML PKDD 2020. LNCS (LNAI), vol. 12459, pp. 279\u2013294. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-67664-3_17"},{"key":"1_CR30","unstructured":"Liu, S., Mocanu, D.C., Pei, Y., Pechenizkiy, M.: Selfish sparse RNN training. In: International Conference on Machine Learning (2021)"},{"key":"1_CR31","unstructured":"Liu, S., Wang, Z.: Ten lessons we have learned in the new \u201cSparseland\u201d: a short handbook for sparse neural network researchers. arXiv preprint arXiv:2302.02596 (2023)"},{"key":"1_CR32","unstructured":"Liu, S., Yin, L., Mocanu, D.C., Pechenizkiy, M.: Do we actually need dense over-parameterization? In-time over-parameterization in sparse training. In: International Conference on Machine Learning (2021)"},{"key":"1_CR33","unstructured":"Liu, S., et al.: Pyraformer: low-complexity pyramidal attention for long-range time series modeling and forecasting. In: International Conference on Learning Representations (2021)"},{"key":"1_CR34","unstructured":"Liu, Y., Wu, H., Wang, J., Long, M.: Non-stationary transformers: rethinking the stationarity in time series forecasting. arXiv preprint arXiv:2205.14415 (2022)"},{"key":"1_CR35","unstructured":"Louizos, C., Welling, M., Kingma, D.P.: Learning sparse neural networks through L_0 regularization. In: International Conference on Learning Representations (2018)"},{"key":"1_CR36","unstructured":"Ma, X., et al.: Effective model sparsification by scheduled grow-and-prune methods. In: International Conference on Learning Representations (2022)"},{"key":"1_CR37","unstructured":"Michel, P., Levy, O., Neubig, G.: Are sixteen heads really better than one? In: Advances in Neural Information Processing Systems, vol. 32 (2019)"},{"key":"1_CR38","doi-asserted-by":"crossref","unstructured":"Mocanu, D.C., Mocanu, E., Nguyen, P.H., Gibescu, M., Liotta, A.: A topological insight into restricted Boltzmann machines. Mach. Learn. 104(2), 243\u2013270 (2016)","DOI":"10.1007\/s10994-016-5570-z"},{"key":"1_CR39","unstructured":"Mocanu, D.C., et al.: Sparse training theory for scalable and efficient agents. In: 20th International Conference on Autonomous Agents and Multiagent Systems (2021)"},{"issue":"1","key":"1_CR40","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41467-018-04316-3","volume":"9","author":"DC Mocanu","year":"2018","unstructured":"Mocanu, D.C., Mocanu, E., Stone, P., Nguyen, P.H., Gibescu, M., Liotta, A.: Scalable training of artificial neural networks with adaptive sparse connectivity inspired by network science. Nat. Commun. 9(1), 1\u201312 (2018)","journal-title":"Nat. Commun."},{"key":"1_CR41","unstructured":"Oreshkin, B.N., Carpov, D., Chapados, N., Bengio, Y.: N-BEATS: neural basis expansion analysis for interpretable time series forecasting. In: International Conference on Learning Representations (2019)"},{"key":"1_CR42","doi-asserted-by":"crossref","unstructured":"Prasanna, S., Rogers, A., Rumshisky, A.: When BERT plays the lottery, all tickets are winning. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.259"},{"key":"1_CR43","doi-asserted-by":"crossref","unstructured":"Qin, Y., Song, D., Cheng, H., Cheng, W., Jiang, G., Cottrell, G.W.: A dual-stage attention-based recurrent neural network for time series prediction. In: International Joint Conference on Artificial Intelligence, pp. 2627\u20132633 (2017)","DOI":"10.24963\/ijcai.2017\/366"},{"key":"1_CR44","doi-asserted-by":"crossref","unstructured":"Rakthanmanon, T., et al.: Searching and mining trillions of time series subsequences under dynamic time warping. In: Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 262\u2013270 (2012)","DOI":"10.1145\/2339530.2339576"},{"issue":"3","key":"1_CR45","doi-asserted-by":"publisher","first-page":"1181","DOI":"10.1016\/j.ijforecast.2019.07.001","volume":"36","author":"D Salinas","year":"2020","unstructured":"Salinas, D., Flunkert, V., Gasthaus, J., Januschowski, T.: DeepAR: probabilistic forecasting with autoregressive recurrent networks. Int. J. Forecast. 36(3), 1181\u20131191 (2020)","journal-title":"Int. J. Forecast."},{"key":"1_CR46","doi-asserted-by":"crossref","unstructured":"Schlake, G.S., Hwel, J.D., Berns, F., Beecks, C.: Evaluating the lottery ticket hypothesis to sparsify neural networks for time series classification. In: International Conference on Data Engineering Workshops (ICDEW), pp. 70\u201373 (2022)","DOI":"10.1109\/ICDEW55742.2022.00015"},{"key":"1_CR47","doi-asserted-by":"crossref","unstructured":"Strubell, E., Ganesh, A., McCallum, A.: Energy and policy considerations for modern deep learning research. In: Proceedings of the AAAI Conference on Artificial Intelligence (2020)","DOI":"10.1609\/aaai.v34i09.7123"},{"key":"1_CR48","doi-asserted-by":"crossref","unstructured":"Subakan, C., Ravanelli, M., Cornell, S., Bronzi, M., Zhong, J.: Attention is all you need in speech separation. In: ICASSP 2021-2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 21\u201325. IEEE (2021)","DOI":"10.1109\/ICASSP39728.2021.9413901"},{"key":"1_CR49","unstructured":"Talagala, T.S., Hyndman, R.J., Athanasopoulos, G., et\u00a0al.: Meta-learning how to forecast time series. Monash Econometrics and Business Statistics Working Papers, vol. 6(18), p. 16 (2018)"},{"issue":"6","key":"1_CR50","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3530811","volume":"55","author":"Y Tay","year":"2022","unstructured":"Tay, Y., Dehghani, M., Bahri, D., Metzler, D.: Efficient transformers: a survey. ACM Comput. Surv. 55(6), 1\u201328 (2022)","journal-title":"ACM Comput. Surv."},{"key":"1_CR51","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Advances in Neural Information Processing Systems (2017)"},{"key":"1_CR52","unstructured":"Wang, Z., Xu, X., Zhang, W., Trajcevski, G., Zhong, T., Zhou, F.: Learning latent seasonal-trend representations for time series forecasting. In: Advances in Neural Information Processing Systems (2022)"},{"key":"1_CR53","unstructured":"Wen, Q., et al.: Transformers in time series: a survey. arXiv preprint arXiv:2202.07125 (2022)"},{"key":"1_CR54","unstructured":"Wolf, T., et\u00a0al.: Transformers: state-of-the-art natural language processing. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pp. 38\u201345 (2020)"},{"key":"1_CR55","unstructured":"Woo, G., Liu, C., Sahoo, D., Kumar, A., Hoi, S.: ETSformer: exponential smoothing transformers for time-series forecasting. arXiv preprint arXiv:2202.01381 (2022)"},{"key":"1_CR56","unstructured":"Wu, H., Hu, T., Liu, Y., Zhou, H., Wang, J., Long, M.: TimesNet: temporal 2D-variation modeling for general time series analysis. In: International Conference on Learning Representations (2023)"},{"key":"1_CR57","first-page":"22419","volume":"34","author":"H Wu","year":"2021","unstructured":"Wu, H., Xu, J., Wang, J., Long, M.: AutoFormer: decomposition transformers with auto-correlation for long-term series forecasting. Adv. Neural. Inf. Process. Syst. 34, 22419\u201322430 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"1_CR58","unstructured":"Xiao, Q., et al.: Dynamic sparse network for time series classification: learning what to \u201csee\u201d. In: Advances in Neural Information Processing Systems (2022)"},{"key":"1_CR59","unstructured":"Yuan, G., et\u00a0al.: MEST: accurate and fast memory-economic sparse training framework on the edge. In: Advances in Neural Information Processing Systems, vol. 34 (2021)"},{"key":"1_CR60","unstructured":"Zeng, A., Chen, M., Zhang, L., Xu, Q.: Are transformers effective for time series forecasting? arXiv preprint arXiv:2205.13504 (2022)"},{"key":"1_CR61","unstructured":"Zhang, T., et al.: Less is more: fast multivariate time series forecasting with light sampling-oriented MLP structures. arXiv preprint arXiv:2207.01186 (2022)"},{"key":"1_CR62","unstructured":"Zhang, Y., Yan, J.: Crossformer: transformer utilizing cross-dimension dependency for multivariate time series forecasting. In: International Conference on Learning Representations (2023)"},{"key":"1_CR63","doi-asserted-by":"crossref","unstructured":"Zhou, H., et al.: Informer: beyond efficient transformer for long sequence time-series forecasting. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a035, pp. 11106\u201311115 (2021)","DOI":"10.1609\/aaai.v35i12.17325"},{"key":"1_CR64","unstructured":"Zhou, T., Ma, Z., Wen, Q., Wang, X., Sun, L., Jin, R.: FEDformer: frequency enhanced decomposed transformer for long-term series forecasting. arXiv preprint arXiv:2201.12740 (2022)"},{"key":"1_CR65","unstructured":"Zhu, M., Gupta, S.: To prune, or not to prune: exploring the efficacy of pruning for model compression. arXiv preprint arXiv:1710.01878 (2017)"}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases. Research Track"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-70341-6_1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,30]],"date-time":"2024-08-30T20:27:16Z","timestamp":1725049636000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-70341-6_1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031703409","9783031703416"],"references-count":65,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-70341-6_1","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"22 August 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Vilnius","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lithuania","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2024.ecmlpkdd.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}