{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T02:48:05Z","timestamp":1780973285686,"version":"3.54.1"},"reference-count":54,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2020,7,25]],"date-time":"2020-07-25T00:00:00Z","timestamp":1595635200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,7,25]],"date-time":"2020-07-25T00:00:00Z","timestamp":1595635200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Process Lett"],"published-print":{"date-parts":[[2020,10]]},"DOI":"10.1007\/s11063-020-10290-z","type":"journal-article","created":{"date-parts":[[2020,7,25]],"date-time":"2020-07-25T16:03:33Z","timestamp":1595693013000},"page":"1123-1153","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":22,"title":["An Analysis of Activation Function Saturation in Particle Swarm Optimization Trained Neural Networks"],"prefix":"10.1007","volume":"52","author":[{"given":"Cody","family":"Dennis","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andries P.","family":"Engelbrecht","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Beatrice M.","family":"Ombuki-Berman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,7,25]]},"reference":[{"key":"10290_CR1","doi-asserted-by":"publisher","unstructured":"Al\u00a0Hazza MH, Adesta EY (2013) Investigation of the effect of cutting speed on the surface roughness parameters in CNC end milling using artificial neural network. In: IOP conference series: materials science and engineering, vol\u00a053, IOP Publishing. https:\/\/doi.org\/10.1088\/1757-899X\/53\/1\/012089","DOI":"10.1088\/1757-899X\/53\/1\/012089"},{"key":"10290_CR2","doi-asserted-by":"crossref","DOI":"10.1093\/oso\/9780198538493.001.0001","volume-title":"Neural networks for pattern recognition","author":"C Bishop","year":"1995","unstructured":"Bishop C (1995) Neural networks for pattern recognition. Oxford University Press, New York"},{"key":"10290_CR3","doi-asserted-by":"crossref","unstructured":"Carvalho M, Ludermir R (2006) Particle swarm optimization of feed-forward neural networks with weight decay. In: Proceedings of the international conference on 3D digital imaging and modeling, pp 1\u20135","DOI":"10.1109\/HIS.2006.264888"},{"key":"10290_CR4","unstructured":"Center NGD (2019) Boulder sunspot number data. https:\/\/www.sws.bom.gov.au\/Educational\/2\/3\/6. Accessed 16 Mar 2019"},{"key":"10290_CR5","doi-asserted-by":"crossref","unstructured":"Dahl G, Sainath T, Hinton G (2013) Improving deep neural networks for LVCSR using rectified linear units and dropout. In: Proceedings of the conference on acoustics, speech and signal processing, pp 8609\u20138613","DOI":"10.1109\/ICASSP.2013.6639346"},{"issue":"4","key":"10290_CR6","doi-asserted-by":"publisher","first-page":"688","DOI":"10.1016\/j.engappai.2009.02.005","volume":"22","author":"M Das","year":"2009","unstructured":"Das M, Dulger L (2009) Signature vecification (SV) toolbox: applications of PSO-NN. Eng Appl Artific Intel 22(4):688\u2013694","journal-title":"Eng Appl Artific Intel"},{"key":"10290_CR7","volume-title":"Neural networks: methodology and applications","author":"G Dreyfus","year":"2005","unstructured":"Dreyfus G (2005) Neural networks: methodology and applications. Springer, Berlin"},{"key":"10290_CR8","unstructured":"Dua D, Graff C (2017) UCI machine learning repository. http:\/\/archive.ics.uci.edu\/ml. Accessed 16 Mar 2019"},{"key":"10290_CR9","unstructured":"Dugas C, Bengio Y, Belisle F, Nadeau C, Garcia R (2001) Incorporating second-order functional knowledge for better option pricing. In: Proceedings of the conference on advances in neural information processing systems, pp 472\u2013478"},{"key":"10290_CR10","doi-asserted-by":"crossref","unstructured":"Eberhart R, Kennedy J (1995) A new optimizer using particle swarm theory. In: Proceedings of the sixth international symposium on micro machine and human science, pp 39\u201343","DOI":"10.1109\/MHS.1995.494215"},{"issue":"1","key":"10290_CR11","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1007\/s10994-017-5683-z","volume":"107","author":"K Eggensperger","year":"2018","unstructured":"Eggensperger K, Lindauer M, Hoos H, Hutter F, Leyton-Brown K (2018) Efficient benchmarking of algorithm configurators via model-based surrogates. Mach Learn 107(1):15\u201341","journal-title":"Mach Learn"},{"key":"10290_CR12","unstructured":"Elliott D (1993) A better activation function for artificial neural networks. Technical report T.R. 93-8, University of Maryland"},{"key":"10290_CR13","doi-asserted-by":"crossref","unstructured":"Engelbrecht A (2012) Particle swarm optimization: velocity initialization. In: Proceedings of the congress on evolutionary computation, pp 1\u20138","DOI":"10.1109\/CEC.2012.6256112"},{"key":"10290_CR14","doi-asserted-by":"crossref","unstructured":"Engelbrecht A (2013) Particle swarm optimization: global best or local best? In: BRICS congress on computational intelligence and 11th Brazilian congress on computational intelligence. IEEE, pp 124\u2013135","DOI":"10.1109\/BRICS-CCI-CBIC.2013.31"},{"key":"10290_CR15","doi-asserted-by":"crossref","unstructured":"Engelbrecht A, Cloete I, Geldenhuys J, Zurada J (1995) Automatic scaling using gamma learning for feedforward neural networks. In: Proceedings of the international workshop on artificial neural networks. Springer, pp 374\u2013381","DOI":"10.1007\/3-540-59497-3_198"},{"key":"10290_CR16","unstructured":"Fisher R (1936) Iris data set. https:\/\/archive.ics.uci.edu\/ml\/datasets\/Iris. Accessed 2 Aug 2018"},{"key":"10290_CR17","unstructured":"Forina M et\u00a0al (1991) Wine data set. https:\/\/archive.ics.uci.edu\/ml\/datasets\/Wine. Accessed 2 Aug 2018"},{"key":"10290_CR18","unstructured":"Glorot X, Bengio Y (2010) Understanding the difficulty of training deep feedforward neural networks. In: International conference on artificial intelligence and statistics, pp 249\u2013256"},{"key":"10290_CR19","first-page":"1756","volume":"13","author":"P Golik","year":"2013","unstructured":"Golik P, Doetsch P, Ney H (2013) Cross-entropy vs. squared error training: a theoretical and experimental comparison. Interspeech 13:1756\u20131760","journal-title":"Interspeech"},{"key":"10290_CR20","doi-asserted-by":"crossref","unstructured":"Gudise V, Venayagamoorthy G (2003) Comparison of particle swarm optimization and backpropagation as training algorithms for neural networks. In: Proceedings of the swarm intelligence symposium, pp 110\u2013117","DOI":"10.1109\/SIS.2003.1202255"},{"key":"10290_CR21","unstructured":"Harrison K (2018) An analysis of parameter control mechanisms for the particle swarm optimization algorithm. Ph.D. thesis, University of Pretoria"},{"key":"10290_CR22","doi-asserted-by":"crossref","unstructured":"Helwig S, Wanka R (2007) Particle swarm optimization in high dimensional bounded search spaces. In: Proceedings of the swarm intelligence symposium, pp 198\u2013205","DOI":"10.1109\/SIS.2007.368046"},{"key":"10290_CR23","doi-asserted-by":"crossref","unstructured":"Helwig S, Wanka R (2008) Theoretical analysis of initial particle swarm behavior. In: Rudolph G, Jansen T, Beume N, Lucas S, Poloni C (eds) Proceedings of the international conference on parallel problem solving from nature, pp 889\u2013898","DOI":"10.1007\/978-3-540-87700-4_88"},{"key":"10290_CR24","doi-asserted-by":"publisher","first-page":"507","DOI":"10.1007\/978-3-642-25566-3_40","volume-title":"Learning and intelligent optimization","author":"F Hutter","year":"2011","unstructured":"Hutter F, Hoos H, Leyton-Brown K (2011) Sequential model-based optimization for general algorithm configuration. In: Coello CA (ed) Learning and intelligent optimization. Springer, Heidelberg, pp 507\u2013523"},{"issue":"4","key":"10290_CR25","doi-asserted-by":"publisher","first-page":"329","DOI":"10.1007\/s13218-015-0381-0","volume":"29","author":"F Hutter","year":"2015","unstructured":"Hutter F, L\u00fccke J, Schmidt-Thieme L (2015) Beyond manual tuning of hyperparameters. KI-K\u00fcnstliche Intelligenz 29(4):329\u2013337","journal-title":"KI-K\u00fcnstliche Intelligenz"},{"key":"10290_CR26","doi-asserted-by":"crossref","unstructured":"Kennedy J, Eberhart R (1995) Particle swarm optimization. In: Proceedings of the international conference on neural networks, vol 4, pp 1942\u20131948","DOI":"10.1109\/ICNN.1995.488968"},{"key":"10290_CR27","doi-asserted-by":"crossref","unstructured":"Kennedy J, Mendes R (2002) Population structure and particle swarm performance. In: Proceedings of the international congress on evolutionary computation, vol 2, pp 1671\u20131676","DOI":"10.1109\/CEC.2002.1004493"},{"key":"10290_CR28","unstructured":"Lawrence S, Tsoi A, Back A (1996) Function approximation with neural networks and local methods: bias, variance and smoothness. In: Proceedings of the Australian conference on neural networks, australian national university, vol 1621"},{"key":"10290_CR29","first-page":"9","volume-title":"Efficient BackProp","author":"Y LeCun","year":"2012","unstructured":"LeCun Y, Bottou L, Orr G, M\u00fcller K (2012) Efficient BackProp. Springer, Berlin, pp 9\u201348"},{"key":"10290_CR30","unstructured":"Maas A, Hannun A, Ng A (2013) Rectifier nonlinearities improve neural network acoustic models. in: Proceedings of the workshop on deep learning for audio, speech, and language processing, vol 30, pp 3\u20138"},{"key":"10290_CR31","doi-asserted-by":"crossref","unstructured":"Mendes R, Cortez P, Rocha M, Neves J (2002) Particle swarms for feedforward neural network training. In: Proceedings of the international joint conference on neural networks, pp 1895\u20131899","DOI":"10.1109\/IJCNN.2002.1007808"},{"key":"10290_CR32","first-page":"950","volume":"4","author":"J Moody","year":"1995","unstructured":"Moody J, Hanson S, Krogh A, Hertz JA (1995) A simple weight decay can improve generalization. Adv Neural Inf Process Syst 4:950\u2013957","journal-title":"Adv Neural Inf Process Syst"},{"key":"10290_CR33","doi-asserted-by":"crossref","unstructured":"Oldewage E (2018) The perils of particle swarm optimization in high dimensional problem spaces. Master\u2019s thesis, Department of Computer Science, University of Pretoria","DOI":"10.1007\/978-3-030-00533-7_27"},{"key":"10290_CR34","doi-asserted-by":"crossref","unstructured":"Olorunda O, Engelbrecht A (2008) Measuring exploration\/exploitation in particle swarms using swarm diversity. In: Proceedings of the international congress on evolutionary computation, pp 1128\u20131134","DOI":"10.1109\/CEC.2008.4630938"},{"issue":"2","key":"10290_CR35","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1162\/neco.1991.3.2.213","volume":"3","author":"J Platt","year":"1991","unstructured":"Platt J (1991) A resource-allocating network for function interpolation. Neural Comput 3(2):213\u2013225","journal-title":"Neural Comput"},{"key":"10290_CR36","unstructured":"Prechelt L (1994) PROBEN1\u2014a set of benchmarks and benchmarking rules for neural network training algorithms. https:\/\/github.com\/jeffheaton\/proben1. Accessed 16 Mar 2019"},{"issue":"3","key":"10290_CR37","first-page":"233","volume":"6","author":"A Rakitianskaia","year":"2012","unstructured":"Rakitianskaia A, Engelbrecht A (2012) Training feedforward neural networks with dynamic particle swarm optimisation. Int J Uncertain Fuzziness Knowl Based Syst 6(3):233\u2013270","journal-title":"Int J Uncertain Fuzziness Knowl Based Syst"},{"key":"10290_CR38","doi-asserted-by":"crossref","unstructured":"Rakitianskaia A, Engelbrecht A (2014) Training high-dimensional neural networks with cooperative particle swarm optimiser. In: Proceedings of the international joint conference on neural networks, pp 4011\u20134018","DOI":"10.1109\/IJCNN.2014.6889933"},{"key":"10290_CR39","doi-asserted-by":"crossref","unstructured":"Rakitianskaia A, Engelbrecht A (2014) Weight regularisation in particle swarm optimisation neural network training. In: Proceedings of the symposium on swarm intelligence, pp 1\u20138","DOI":"10.1109\/SIS.2014.7011773"},{"key":"10290_CR40","doi-asserted-by":"crossref","unstructured":"Rakitianskaia A, Engelbrecht A (2015) Measuring saturation in neural networks. In: Proceedings of the symposium series on computational intelligence, pp 1423\u20131430","DOI":"10.1109\/SSCI.2015.202"},{"key":"10290_CR41","doi-asserted-by":"crossref","unstructured":"Rakitianskaia A, Engelbrecht A (2015) Saturation in PSO neural network training: good or evil? In: Proceedings of the international congress on evolutionary computation, pp 125\u2013132","DOI":"10.1109\/CEC.2015.7256883"},{"issue":"1","key":"10290_CR42","first-page":"19","volume":"14","author":"DP Rini","year":"2011","unstructured":"Rini DP, Shamsuddin SM, Yuhaniz SS (2011) Particle swarm optimization: technique, system and challenges. Int J Comput Appl \u2018 14(1):19\u201327","journal-title":"Int J Comput Appl \u2018"},{"key":"10290_CR43","doi-asserted-by":"crossref","unstructured":"R\u00f6bel A (1994) The dynamic pattern selection algorithm: effective training and controlled generalization of backpropagation neural networks. Technical report, Technische Universit\u00e4t Berlin","DOI":"10.1007\/978-1-4471-2097-1_151"},{"key":"10290_CR44","doi-asserted-by":"publisher","unstructured":"Shi Y, Eberhart R (1998) A modified particle swarm optimizer. In: Proceedings of the international congress on evolutionary computation, pp 69\u201373. https:\/\/doi.org\/10.1109\/ICEC.1998.699146","DOI":"10.1109\/ICEC.1998.699146"},{"key":"10290_CR45","first-page":"541","volume-title":"Automated design of metaheuristic algorithms","author":"T St\u00fctzle","year":"2019","unstructured":"St\u00fctzle T, L\u00f3pez-Ib\u00e1\u00f1ez M (2019) Automated design of metaheuristic algorithms. Springer, Cham, pp 541\u2013579"},{"key":"10290_CR46","first-page":"84","volume":"26","author":"F van den Bergh","year":"2000","unstructured":"van den Bergh F, Engelbrecht A (2000) Cooperative learning in neural networks using particle swarm optimizers. S Afr Comput J 26:84\u201390","journal-title":"S Afr Comput J"},{"key":"10290_CR47","doi-asserted-by":"crossref","unstructured":"van Wyk A, Engelbrecht A (2010) Overfitting by PSO trained feedforward neural networks. In: Proceedings of the international congress on evolutionary computation, pp 1\u20138","DOI":"10.1109\/CEC.2010.5586333"},{"key":"10290_CR48","doi-asserted-by":"crossref","unstructured":"van Wyk A, Engelbrecht A (2016) Analysis of activation functions for particle swarm optimized feedforward neural networks. In: Proceedings of the international congress on evolutionary computation, pp 423\u2013430","DOI":"10.1109\/CEC.2016.7743825"},{"key":"10290_CR49","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1007\/978-3-319-41000-5_37","volume-title":"Advances in swarm intelligence","author":"A Volschenk","year":"2016","unstructured":"Volschenk A, Engelbrecht A (2016) An analysis of competitive coevolutionary particle swarm optimizers to train neural network game tree evaluation functions. In: Tan Y, Shi Y, Niu B (eds) Advances in swarm intelligence. Springer, Cham, pp 369\u2013380"},{"key":"10290_CR50","unstructured":"Werbos PJ (1974) Beyond regression: new tools for prediction and analysis in the behavioural sciences. Ph.D. thesis, Harvard University"},{"issue":"6","key":"10290_CR51","doi-asserted-by":"publisher","first-page":"899","DOI":"10.1109\/72.165592","volume":"3","author":"L Wessels","year":"1992","unstructured":"Wessels L, Barnard E (1992) Avoiding false local minima by proper initialization of connections. Trans Neural Netw 3(6):899\u2013905","journal-title":"Trans Neural Netw"},{"key":"10290_CR52","unstructured":"Wolberg W (1990) Breast cancer wisconsin (original) data set. https:\/\/archive.ics.uci.edu\/ml\/datasets\/Breast+Cancer+Wisconsin+%28Original%29. Accessed 2 Aug 2018"},{"key":"10290_CR53","unstructured":"Wyk A, Engelbrecht A (2011) Lambda-gamma learning with feedforward neural networks using particle swarm optimization. In: Proceedings of the symposium on swarm intelligence, pp 1\u20138"},{"issue":"1","key":"10290_CR54","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1049\/iet-ifs.2015.0211","volume":"11","author":"X Xiao","year":"2017","unstructured":"Xiao X, Wang Z, Li Q, Xia S, Jiang Y (2017) Back-propagation neural network on markov chains from system call sequences: a new approach for detecting android malware with system call sequences. IET Inf Secur 11(1):8\u201315","journal-title":"IET Inf Secur"}],"container-title":["Neural Processing Letters"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11063-020-10290-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11063-020-10290-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11063-020-10290-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,10]],"date-time":"2024-08-10T20:08:28Z","timestamp":1723320508000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11063-020-10290-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,7,25]]},"references-count":54,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2020,10]]}},"alternative-id":["10290"],"URL":"https:\/\/doi.org\/10.1007\/s11063-020-10290-z","relation":{},"ISSN":["1370-4621","1573-773X"],"issn-type":[{"value":"1370-4621","type":"print"},{"value":"1573-773X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,7,25]]},"assertion":[{"value":"25 July 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}