{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,20]],"date-time":"2026-08-20T15:33:39Z","timestamp":1787240019439,"version":"build-2736575974"},"reference-count":58,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2017,10,19]],"date-time":"2017-10-19T00:00:00Z","timestamp":1508371200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2019,7]]},"DOI":"10.1007\/s00521-017-3210-6","type":"journal-article","created":{"date-parts":[[2017,10,19]],"date-time":"2017-10-19T14:18:57Z","timestamp":1508422737000},"page":"2507-2521","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":138,"title":["A comparative performance analysis of different activation functions in LSTM networks for classification"],"prefix":"10.1007","volume":"31","author":[{"given":"Amir","family":"Farzad","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0080-2743","authenticated-orcid":false,"given":"Hoda","family":"Mashayekhi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hamid","family":"Hassanpour","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,10,19]]},"reference":[{"key":"3210_CR1","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter S, Schmidhuber J (1997) Long short-term memory. Neural Comput 9:1735\u20131780. doi: 10.1162\/neco.1997.9.8.1735","journal-title":"Neural Comput"},{"key":"3210_CR2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-24797-2","volume-title":"Supervised sequence labelling with recurrent neural networks","author":"A Graves","year":"2012","unstructured":"Graves A (2012) Supervised sequence labelling with recurrent neural networks. Springer, Berlin"},{"issue":"5","key":"3210_CR3","doi-asserted-by":"publisher","first-page":"602","DOI":"10.1016\/j.neunet.2005.06.042","volume":"18","author":"A Graves","year":"2005","unstructured":"Graves A, Schmidhuber J (2005) Framewise phoneme classification with bidirectional lstm and other neural network architectures. Neural Netw 18(5):602\u2013610","journal-title":"Neural Netw"},{"key":"3210_CR4","unstructured":"Liwicki M, Graves A, Bunke H, Schmidhuber J (2007) A novel approach to on-line handwriting recognition based on bidirectional long short-term memory networks. In: Proceedings of the 9th international conference on document analysis and recognition, ICDAR 2007"},{"key":"3210_CR5","doi-asserted-by":"publisher","first-page":"855","DOI":"10.1109\/TPAMI.2008.137","volume":"31","author":"A Graves","year":"2009","unstructured":"Graves A, Liwicki M, Fern\u00e1ndez S et al (2009) A novel connectionist system for unconstrained handwriting recognition. IEEE Trans Pattern Anal Mach Intell 31:855\u2013868. doi: 10.1109\/TPAMI.2008.137","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"3210_CR6","doi-asserted-by":"crossref","first-page":"799","DOI":"10.1007\/11550907_126","volume-title":"Artificial neural networks: formal models and their applications\u2014ICANN 2005","author":"A Graves","year":"2005","unstructured":"Graves A, Fern\u00e1ndez S, Schmidhuber J (2005) Bidirectional LSTM Networks for Improved Phoneme Classification and Recognition. In: Duch W, Kacprzyk J, Oja E, Zadro\u017cny S (eds) Artificial neural networks: formal models and their applications\u2014ICANN 2005. Springer, Berlin, pp 799\u2013804"},{"key":"3210_CR7","doi-asserted-by":"crossref","unstructured":"Otte S, Krechel D, Liwicki M, Dengel A (2012) Local feature based online mode detection with recurrent neural networks. In: 2012 international conference on frontiers in handwriting recognition (ICFHR). pp 533\u2013537","DOI":"10.1109\/ICFHR.2012.229"},{"key":"3210_CR8","doi-asserted-by":"crossref","unstructured":"Graves A, Mohamed A, Hinton G (2013) Speech recognition with deep recurrent neural networks. arXiv:1303.5778 [cs]","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"3210_CR9","doi-asserted-by":"publisher","unstructured":"Thang Luong IS (2014) Addressing the rare word problem in neural machine translation. doi: 10.3115\/v1\/P15-1002","DOI":"10.3115\/v1\/P15-1002"},{"key":"3210_CR10","doi-asserted-by":"crossref","unstructured":"W\u00f6llmer M, Metallinou A, Eyben F, et al (2010) Context-sensitive multimodal emotion recognition from speech and facial expression using bidirectional LSTM modeling. In: Proceedings of interspeech, Makuhari. pp 2362\u20132365","DOI":"10.21437\/Interspeech.2010-646"},{"key":"3210_CR11","doi-asserted-by":"crossref","unstructured":"Sak H, Senior A, Beaufays F (2014) Long short-term memory recurrent neural network architectures for large scale acoustic modeling. In Proceedings of the annual conference of international speech communication association (INTERSPEECH). pp 338\u2013342","DOI":"10.21437\/Interspeech.2014-80"},{"key":"3210_CR12","doi-asserted-by":"crossref","unstructured":"Fan Y, Qian Y, Xie F, Soong FK (2014) TTS synthesis with bidirectional LSTM based recurrent neural networks. In: Proceedings interspeech. pp 1964\u20131968","DOI":"10.21437\/Interspeech.2014-443"},{"key":"3210_CR13","unstructured":"Zaremba W, Sutskever I, Vinyals O (2014) Recurrent neural network regularization. arXiv:1409.2329 [cs]"},{"key":"3210_CR14","unstructured":"S\u00f8nderby SK, Winther O (2014) Protein secondary structure prediction with long short term memory networks. arXiv:1412.7828 [cs, q-bio]"},{"key":"3210_CR15","doi-asserted-by":"crossref","unstructured":"Marchi E, Ferroni G, Eyben F, et al (2014) Multi-resolution linear prediction based features for audio onset detection with bidirectional LSTM neural networks. In: 2014 IEEE international conference on acoustics, speech and signal processing (ICASSP). pp 2164\u20132168","DOI":"10.1109\/ICASSP.2014.6853982"},{"key":"3210_CR16","doi-asserted-by":"crossref","unstructured":"Donahue J, Hendricks LA, Guadarrama S, et al (2014) Long-term recurrent convolutional networks for visual recognition and description. arXiv:1411.4389 [cs]","DOI":"10.21236\/ADA623249"},{"key":"3210_CR17","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1109\/TITS.2011.2119483","volume":"12","author":"M Wollmer","year":"2011","unstructured":"Wollmer M, Blaschke C, Schindl T et al (2011) Online driver distraction detection using long short-term memory. IEEE Trans Intell Transp Syst 12:574\u2013582. doi: 10.1109\/TITS.2011.2119483","journal-title":"IEEE Trans Intell Transp Syst"},{"key":"3210_CR18","doi-asserted-by":"publisher","first-page":"6438","DOI":"10.1016\/j.eswa.2013.05.053","volume":"40","author":"GSS Gomes da","year":"2013","unstructured":"da Gomes GSS, Ludermir TB (2013) Optimization of the weights and asymmetric activation function family of neural network for time series forecasting. Exp Syst Appl 40:6438\u20136446. doi: 10.1016\/j.eswa.2013.05.053","journal-title":"Exp Syst Appl"},{"key":"3210_CR19","first-page":"163","volume":"2","author":"W Duch","year":"1999","unstructured":"Duch W, Jankowski N (1999) Survey of neural transfer functions. Neural Comput Surv 2:163\u2013213","journal-title":"Neural Comput Surv"},{"key":"3210_CR20","doi-asserted-by":"publisher","first-page":"183","DOI":"10.1016\/S0165-1889(02)00157-4","volume":"28","author":"S Singh Sodhi","year":"2003","unstructured":"Singh Sodhi S, Chandra P (2003) A class +1 sigmoidal activation functions for FFANNs. J Econ Dyna Control 28:183\u2013187","journal-title":"J Econ Dyna Control"},{"key":"3210_CR21","doi-asserted-by":"publisher","first-page":"417","DOI":"10.1007\/s00521-010-0407-3","volume":"20","author":"GSS Gomes da","year":"2010","unstructured":"da Gomes GSS, Ludermir TB, Lima LMMR (2010) Comparison of new activation functions in neural network for forecasting financial time series. Neural Comput Appl 20:417\u2013439. doi: 10.1007\/s00521-010-0407-3","journal-title":"Neural Comput Appl"},{"key":"3210_CR22","unstructured":"Gomes GS d S, Ludermir TB (2008) Complementary log-log and probit: activation functions implemented in artificial neural networks. In: Eighth international conference on hybrid intelligent systems, 2008. HIS\u201908. pp 939\u2013942"},{"key":"3210_CR23","doi-asserted-by":"publisher","first-page":"3606","DOI":"10.1103\/PhysRevE.58.3606","volume":"58","author":"MB Michal Rosen-Zvi","year":"1998","unstructured":"Michal Rosen-Zvi MB (1998) Learnability of periodic activation functions: general results. Phys Rev E 58:3606\u20133609. doi: 10.1103\/PhysRevE.58.3606","journal-title":"Phys Rev E"},{"key":"3210_CR24","doi-asserted-by":"publisher","first-page":"928","DOI":"10.1162\/neco.1993.5.6.928","volume":"5","author":"H Leung","year":"1993","unstructured":"Leung H, Haykin S (1993) Rational function neural network. Neural Comput 5:928\u2013938. doi: 10.1162\/neco.1993.5.6.928","journal-title":"Neural Comput"},{"key":"3210_CR25","doi-asserted-by":"publisher","first-page":"821","DOI":"10.1109\/TNN.2005.851786","volume":"16","author":"L Ma","year":"2005","unstructured":"Ma L, Khorasani K (2005) Constructive feedforward neural networks using Hermite polynomial activation functions. IEEE Trans Neural Netw 16:821\u2013833. doi: 10.1109\/TNN.2005.851786","journal-title":"IEEE Trans Neural Netw"},{"key":"3210_CR26","doi-asserted-by":"publisher","first-page":"1069","DOI":"10.1016\/S0893-6080(09)80018-X","volume":"6","author":"K Hornik","year":"1993","unstructured":"Hornik K (1993) Some new results on neural network approximation. Neural Netw 6:1069\u20131072. doi: 10.1016\/S0893-6080(09)80018-X","journal-title":"Neural Netw"},{"key":"3210_CR27","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1016\/0893-6080(91)90009-T","volume":"4","author":"K Hornik","year":"1991","unstructured":"Hornik K (1991) Approximation capabilities of multilayer feedforward networks. Neural Netw 4:251\u2013257. doi: 10.1016\/0893-6080(91)90009-T","journal-title":"Neural Netw"},{"key":"3210_CR28","doi-asserted-by":"publisher","first-page":"210","DOI":"10.1162\/neco.1990.2.2.210","volume":"2","author":"E Hartman","year":"1990","unstructured":"Hartman E, Keeler JD, Kowalski JM (1990) Layered neural networks with Gaussian hidden units as universal approximations. Neural Comput 2:210\u2013215. doi: 10.1162\/neco.1990.2.2.210","journal-title":"Neural Comput"},{"key":"3210_CR29","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1023\/B:JINT.0000049175.78893.2f","volume":"41","author":"EN Skoundrianos","year":"2004","unstructured":"Skoundrianos EN, Tzafestas SG (2004) Modelling and FDI of dynamic discrete time systems using a MLP with a new sigmoidal activation function. J Intell Robot Syst 41:19\u201336. doi: 10.1023\/B:JINT.0000049175.78893.2f","journal-title":"J Intell Robot Syst"},{"key":"3210_CR30","volume-title":"Adaptive pattern recognition and neural networks","author":"Y-H Pao","year":"1989","unstructured":"Pao Y-H (1989) Adaptive pattern recognition and neural networks. Addison-Wesley Longman Publishing Co. Inc, Boston"},{"key":"3210_CR31","unstructured":"Carroll SM, Dickinson BW (1989) Construction of neural nets using the radon transform. In: International joint conference on neural networks, 1989, vol 1. IJCNN. pp 607\u2013611"},{"key":"3210_CR32","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1007\/BF02551274","volume":"2","author":"G Cybenko","year":"1989","unstructured":"Cybenko G (1989) Approximation by superpositions of a sigmoidal function. Math Control Signal Syst 2:303\u2013314. doi: 10.1007\/BF02551274","journal-title":"Math Control Signal Syst"},{"key":"3210_CR33","doi-asserted-by":"publisher","first-page":"1350","DOI":"10.1109\/TNN.2004.831198","volume":"15","author":"P Chandra","year":"2004","unstructured":"Chandra P, Singh Y (2004) Feedforward sigmoidal networks\u2014equicontinuity and fault-tolerance properties. IEEE Trans Neural Netw 15:1350\u20131366. doi: 10.1109\/TNN.2004.831198","journal-title":"IEEE Trans Neural Netw"},{"key":"3210_CR34","first-page":"433","volume-title":"Back-propagation: theory, architectures and applications","author":"RJ Williams","year":"1995","unstructured":"Williams RJ, Zipser D (1995) Gradient-based learning algorithms for recurrent networks and their computational complexity. In: Chauvin Y, Rumelhart DE (eds) Back-propagation: theory, architectures and applications. L. Erlbaum Associates Inc., Hillsdale, pp 433\u2013486"},{"key":"3210_CR35","unstructured":"Zeiler MD (2012) ADADELTA: an adaptive learning rate method. arXiv:1212.5701 [cs]"},{"key":"3210_CR36","doi-asserted-by":"publisher","first-page":"182","DOI":"10.1016\/j.neucom.2014.06.007","volume":"143","author":"S Singh Sodhi","year":"2014","unstructured":"Singh Sodhi S, Chandra P (2014) Bi-modal derivative activation function for sigmoidal feedforward networks. Neurocomputing 143:182\u2013196. doi: 10.1016\/j.neucom.2014.06.007","journal-title":"Neurocomputing"},{"key":"3210_CR37","doi-asserted-by":"crossref","unstructured":"Yuan M, Hu H, Jiang Y, Hang S (2013) A new camera calibration based on neural network with tunable activation function in intelligent space. In: 2013 6th international symposium on computational intelligence and design (ISCID). pp 371\u2013374","DOI":"10.1109\/ISCID.2013.99"},{"key":"3210_CR38","doi-asserted-by":"crossref","unstructured":"Chandra P, Sodhi SS (2014) A skewed derivative activation function for SFFANNs. In: Recent advances and innovations in engineering (ICRAIE). IEEE, pp 1\u20136","DOI":"10.1109\/ICRAIE.2014.6909324"},{"key":"3210_CR39","unstructured":"Elliott DL (1993) A better activation function for artificial neural networks. Technical Report ISR TR 93\u20138, University of Maryland"},{"key":"3210_CR40","unstructured":"Hara K, Nakayamma K (1994) Comparison of activation functions in multilayer neural network for pattern classification. In: 1994 IEEE international conference on neural networks, 1994. IEEE world congress on computational intelligence, vol 5. pp 2997\u20133002"},{"key":"3210_CR41","doi-asserted-by":"crossref","unstructured":"Burhani H, Feng W, Hu G (2015) Denoising autoencoder in neural networks with modified Elliott activation function and sparsity-favoring cost function. In: 2015 3rd international conference on applied computing and information technology\/2nd international conference on computational science and intelligence (ACIT-CSI). pp 343\u2013348","DOI":"10.1109\/ACIT-CSI.2015.67"},{"key":"3210_CR42","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1016\/j.neucom.2003.08.005","volume":"56","author":"P Chandra","year":"2004","unstructured":"Chandra P, Singh Y (2004) A case for the self-adaptation of activation functions in FFANNs. Neurocomputing 56:447\u2013454. doi: 10.1016\/j.neucom.2003.08.005","journal-title":"Neurocomputing"},{"key":"3210_CR43","unstructured":"Nair V, Hinton GE (2010) Rectified linear units improve restricted boltzmann machines. In: Proceedings of the 27th international conference on machine learning (ICML 2010). pp 807\u2013814"},{"key":"3210_CR44","first-page":"26","volume":"4","author":"T Tieleman","year":"2012","unstructured":"Tieleman T, Hinton G (2012) Lecture 6.5-rmsprop: divide the gradient by a running average of its recent magnitude. COURSERA Neural Netw Mach Learn 4:26\u201331","journal-title":"COURSERA Neural Netw Mach Learn"},{"key":"3210_CR45","first-page":"2121","volume":"12","author":"J Duchi","year":"2011","unstructured":"Duchi J, Hazan E, Singer Y (2011) Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. Journal of Machine Learning Research 12:2121\u20132159","journal-title":"Journal of Machine Learning Research"},{"key":"3210_CR46","unstructured":"Lipton ZC, Berkowitz J, Elkan C (2015) A critical review of recurrent neural networks for sequence learning. arXiv:1506.00019 [cs]"},{"key":"3210_CR47","unstructured":"Ruder S (2016) An overview of gradient descent optimization algorithms. arXiv:1609.04747 [cs]"},{"key":"3210_CR48","unstructured":"Hinton GE, Srivastava N, Krizhevsky A, et al (2012) Improving neural networks by preventing co-adaptation of feature detectors, vol abs\/1207.0580. arXiv preprint arXiv:1207.0580 . The Computing Research Repository (CoRR)"},{"key":"3210_CR49","doi-asserted-by":"crossref","unstructured":"Pang B, Lee L (2005) Seeing stars: exploiting class relationships for sentiment categorization with respect to rating scales. In: Proceedings of the 43rd annual meeting on association for computational linguistics. Association for Computational Linguistics, Stroudsburg, PA, USA. pp 115\u2013124","DOI":"10.3115\/1219840.1219855"},{"key":"3210_CR50","unstructured":"Maas AL, Daly RE, Pham PT, et al (2011) Learning word vectors for sentiment analysis. In: Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies, vol 1. Association for Computational Linguistics, Stroudsburg, PA, USA. pp 142\u2013150"},{"key":"3210_CR51","unstructured":"Lenc L, Hercig T (2016) Neural networks for sentiment analysis in Czech. In: ITAT 2016 proceedings, CEUR Workshop Proceedings, vol 1649. pp 48\u201355"},{"key":"3210_CR52","first-page":"3079","volume-title":"Advances in neural information processing systems","author":"AM Dai","year":"2015","unstructured":"Dai AM, Le QV (2015) Semi-supervised sequence learning. In: Cortes C, Lawrence ND, Lee DD et al (eds) Advances in neural information processing systems, vol 28. Curran Associates Inc, Red Hook, pp 3079\u20133087"},{"key":"3210_CR53","unstructured":"Arjovsky M, Shah A, Bengio Y (2016) Unitary evolution recurrent neural networks. In: Proceedings of the 33rd international conference on machine learning. pp 1120\u20131128"},{"key":"3210_CR54","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1109\/72.279181","volume":"5","author":"Y Bengio","year":"1994","unstructured":"Bengio Y, Simard P, Frasconi P (1994) Learning long-term dependencies with gradient descent is difficult. IEEE Trans Neural Netw 5:157\u2013166. doi: 10.1109\/72.279181","journal-title":"IEEE Trans Neural Netw"},{"key":"3210_CR55","unstructured":"Pascanu R, Mikolov T, Bengio Y (2013) On the difficulty of training recurrent neural networks. In: Proceedings of the 30th international conference on machine learning. pp 1310\u20131318"},{"key":"3210_CR56","unstructured":"Gers FA, Schmidhuber J (2000) Recurrent nets that time and count. In: Proceedings of the IEEE-INNS-ENNS international joint conference on neural networks. IJCNN 2000. Neural computing: new challenges and perspectives for the new millennium, vol 3. pp 189\u2013194"},{"key":"3210_CR57","doi-asserted-by":"publisher","first-page":"2673","DOI":"10.1109\/78.650093","volume":"45","author":"M Schuster","year":"1997","unstructured":"Schuster M, Paliwal KK (1997) Bidirectional recurrent neural networks. IEEE Trans Signal Process 45:2673\u20132681. doi: 10.1109\/78.650093","journal-title":"IEEE Trans Signal Process"},{"key":"3210_CR58","doi-asserted-by":"crossref","unstructured":"Cho K, van Merrienboer B, Bahdanau D, Bengio Y (2014) On the properties of neural machine translation: encoder\u2013decoder approaches. arXiv:1409.1259 [cs, stat]","DOI":"10.3115\/v1\/W14-4012"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-017-3210-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00521-017-3210-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-017-3210-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,4]],"date-time":"2022-08-04T18:14:20Z","timestamp":1659636860000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00521-017-3210-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,10,19]]},"references-count":58,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2019,7]]}},"alternative-id":["3210"],"URL":"https:\/\/doi.org\/10.1007\/s00521-017-3210-6","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,10,19]]},"assertion":[{"value":"3 August 2016","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 October 2017","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 October 2017","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"We declare that we do not have any commercial or associative interest that represents a conflict of interest in connection with the work submitted.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}