{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T00:21:08Z","timestamp":1759191668851,"version":"3.44.0"},"publisher-location":"Cham","reference-count":27,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032060778","type":"print"},{"value":"9783032060785","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T00:00:00Z","timestamp":1759190400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T00:00:00Z","timestamp":1759190400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-06078-5_21","type":"book-chapter","created":{"date-parts":[[2025,9,29]],"date-time":"2025-09-29T18:50:10Z","timestamp":1759171810000},"page":"362-379","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["The Vanishing Empirical Variance in\u00a0Randomly Initialized Deep ReLU Networks"],"prefix":"10.1007","author":[{"given":"Micha\u0142","family":"Grzejdziak-Zdziarski","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David M. J.","family":"Tax","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Marco","family":"Loog","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,9,30]]},"reference":[{"key":"21_CR1","unstructured":"Bachlechner, T., Majumder, B.P., Mao, H., Cottrell, G., McAuley, J.: Rezero is all you need: fast convergence at large depth. In: de\u00a0Campos, C., Maathuis, M.H. (eds.) Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence. vol.\u00a0161, pp. 1352\u20131361. PMLR 2021). https:\/\/proceedings.mlr.press\/v161\/bachlechner21a.html"},{"key":"21_CR2","unstructured":"Bartlett, P., Helmbold, D., Long, P.: Gradient descent with identity initialization efficiently learns positive definite linear transformations by deep residual networks. In: Dy, J., Krause, A. (eds.) Proceedings of the 35th International Conference on Machine Learning. vol.\u00a080, pp. 521\u2013530 (2018). https:\/\/proceedings.mlr.press\/v80\/bartlett18a.html"},{"issue":"2","key":"21_CR3","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1109\/72.279181","volume":"5","author":"Y Bengio","year":"1994","unstructured":"Bengio, Y., Simard, P., Frasconi, P.: Learning long-term dependencies with gradient descent is difficult. IEEE Trans. Neural Networks 5(2), 157\u2013166 (1994). https:\/\/doi.org\/10.1109\/72.279181","journal-title":"IEEE Trans. Neural Networks"},{"key":"21_CR4","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780198538493.001.0001","volume-title":"Neural Networks for Pattern Recognition","author":"CM Bishop","year":"1995","unstructured":"Bishop, C.M.: Neural Networks for Pattern Recognition. OUP, USA (1995)"},{"key":"21_CR5","unstructured":"Burkholz, R., Dubatovka, A.: Initialization of Relus for dynamical isometry. In: Wallach, H., Larochelle, H., Beygelzimer, A., d\u2019Alch\u00e9-Buc, F., Fox, E., Garnett, R. (eds.) Advances in Neural Information Processing Systems. vol.\u00a032. Curran Associates, Inc. (2019). https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/d9731321ef4e063ebbee79298fa36f56-Paper.pdf"},{"key":"21_CR6","unstructured":"Dauphin, Y.N., Schoenholz, S.: Metainit: initializing learning by learning to initialize. In: Wallach, H., Larochelle, H., Beygelzimer, A., d\u2019Alch\u00e9-Buc, F., Fox, E., Garnett, R. (eds.) Advances in Neural Information Processing Systems. vol.\u00a032. Curran Associates, Inc. (2019). https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/876e8108f87eb61877c6263228b67256-Paper.pdf"},{"key":"21_CR7","unstructured":"Du, S., Hu, W.: Width provably matters in optimization for deep linear neural networks. In: Chaudhuri, K., Salakhutdinov, R. (eds.) Proceedings of the 36th International Conference on Machine Learning. vol.\u00a097, pp. 1655\u20131664. PMLR (2019). https:\/\/proceedings.mlr.press\/v97\/du19a.html"},{"key":"21_CR8","unstructured":"Glorot, X., Bengio, Y.: Understanding the difficulty of training deep feedforward neural networks. In: Teh, Y.W., Titterington, M. (eds.) Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics, vol.\u00a09, pp. 249\u2013256. PMLR, Chia Laguna Resort, Sardinia, Italy (2010). https:\/\/proceedings.mlr.press\/v9\/glorot10a.html"},{"key":"21_CR9","unstructured":"Hanin, B.: Which neural net architectures give rise to exploding and vanishing gradients? In: Bengio, S., Wallach, H., Larochelle, H., Grauman, K., Cesa-Bianchi, N., Garnett, R. (eds.) Advances in Neural Information Processing Systems. vol.\u00a031. Curran Associates, Inc. (2018). https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/13f9896df61279c928f19721878fac41-Paper.pdf"},{"key":"21_CR10","unstructured":"Hanin, B., Rolnick, D.: How to start training: the effect of initialization and architecture. In: Bengio, S., Wallach, H., Larochelle, H., Grauman, K., Cesa-Bianchi, N., Garnett, R. (eds.) Advances in Neural Information Processing Systems. vol.\u00a031. Curran Associates, Inc. (2018). https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/d81f9c1be2e08964bf9f24b15f0e4900-Paper.pdf"},{"key":"21_CR11","doi-asserted-by":"publisher","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Delving deep into rectifiers: surpassing human-level performance on Imagenet classification. In: 2015 IEEE ICCV, pp. 1026\u20131034 (2015). https:\/\/doi.org\/10.1109\/ICCV.2015.123","DOI":"10.1109\/ICCV.2015.123"},{"key":"21_CR12","doi-asserted-by":"publisher","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778 (2016). https:\/\/doi.org\/10.1109\/CVPR.2016.90","DOI":"10.1109\/CVPR.2016.90"},{"key":"21_CR13","first-page":"1","volume":"1","author":"S Hochreiter","year":"1991","unstructured":"Hochreiter, S.: Untersuchungen zu dynamischen neuronalen netzen. Master\u2019s thesis, Institut fur Informatik, Technische Universitat, Munchen 1, 1\u2013150 (1991)","journal-title":"Master\u2019s thesis, Institut fur Informatik, Technische Universitat, Munchen"},{"key":"21_CR14","volume-title":"Matrix Analysis","author":"RA Horn","year":"2013","unstructured":"Horn, R.A., Johnson, C.R.: Matrix Analysis, 2nd edn. Cambridge University Press, Cambridge (2013)","edition":"2"},{"key":"21_CR15","unstructured":"Hu, W., Xiao, L., Pennington, J.: Provable benefit of orthogonal initialization in optimizing deep linear networks. In: International Conference on Learning Representations (2020). https:\/\/openreview.net\/forum?id=rkgqN1SYvr"},{"key":"21_CR16","unstructured":"Kingma, D.P., Ba, J.: Adam: A method for stochastic optimization. In: International Conference on Learning Representations (2015)"},{"key":"21_CR17","unstructured":"Krizhevsky, A.: Learning multiple layers of features from tiny images. Technical Report, University of Toronto (2009). https:\/\/www.cs.toronto.edu\/~kriz\/learning-features-2009-TR.pdf"},{"issue":"11","key":"21_CR18","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun, Y., Bottou, L., Bengio, Y., Haffner, P.: Gradient-based learning applied to document recognition. Proc. IEEE 86(11), 2278\u20132324 (1998). https:\/\/doi.org\/10.1109\/5.726791","journal-title":"Proc. IEEE"},{"key":"21_CR19","doi-asserted-by":"publisher","unstructured":"LeCun, Y., Bottou, L., Orr, G.B., M\u00fcller, K.R.: Efficient backprop. In: Orr, G.B., M\u00fcller, K.R. (eds.) Neural Networks: Tricks of the Trade, pp. 9\u201350. Springer Berlin Heidelberg, Berlin, Heidelberg (1998). https:\/\/doi.org\/10.1007\/3-540-49430-8_2","DOI":"10.1007\/3-540-49430-8_2"},{"key":"21_CR20","unstructured":"Mishkin, D., Matas, J.: All you need is a good init. In: International Conference on Learning Representations (2016)"},{"key":"21_CR21","doi-asserted-by":"publisher","unstructured":"O\u2019Neill, B.: Some useful moment results in sampling problems. American Stat. 68(4), 282\u2013296 (2014). https:\/\/doi.org\/10.1080\/00031305.2014.966589","DOI":"10.1080\/00031305.2014.966589"},{"key":"21_CR22","unstructured":"Saxe, A., McClelland, J., Ganguli, S.: Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. In: International Conference on Learning Representations (2014)"},{"key":"21_CR23","unstructured":"Shamir, O.: Exponential convergence time of gradient descent for one-dimensional deep linear neural networks. In: Beygelzimer, A., Hsu, D. (eds.) Proceedings of the Thirty-Second Conference on Learning Theory. vol.\u00a099, pp. 2691\u20132713. PMLR (2019). https:\/\/proceedings.mlr.press\/v99\/shamir19a.html"},{"key":"21_CR24","unstructured":"Vladimirova, M., Verbeek, J., Mesejo, P., Arbel, J.: Understanding priors in Bayesian neural networks at the unit level. In: Chaudhuri, K., Salakhutdinov, R. (eds.) Proceedings of the 36th International Conference on Machine Learning. vol.\u00a097, pp. 6458\u20136467. PMLR (2019). https:\/\/proceedings.mlr.press\/v97\/vladimirova19a.html"},{"key":"21_CR25","unstructured":"Xiao, L., Bahri, Y., Sohl-Dickstein, J., Schoenholz, S., Pennington, J.: Dynamical isometry and a mean field theory of CNNs: How to train 10,000-layer vanilla convolutional neural networks. In: Dy, J., Krause, A. (eds.) Proceedings of the 35th International Conference on Machine Learning. vol.\u00a080, pp. 5393\u20135402. PMLR (2018). https:\/\/proceedings.mlr.press\/v80\/xiao18a.html"},{"key":"21_CR26","unstructured":"Zhang, H., Dauphin, Y.N., Ma, T.: Residual learning without normalization via better initialization. In: International Conference on Learning Representations (2019). https:\/\/openreview.net\/forum?id=H1gsz30cKX"},{"key":"21_CR27","unstructured":"Zhao, J., Schaefer, F.T., Anandkumar, A.: Zero initialization: initializing neural networks with only zeros and ones. Trans. Mach. Learn. Res. (2022). https:\/\/openreview.net\/forum?id=1AxQpKmiTc"}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases. Research Track"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-06078-5_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,29]],"date-time":"2025-09-29T18:50:20Z","timestamp":1759171820000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-06078-5_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,30]]},"ISBN":["9783032060778","9783032060785"],"references-count":27,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-06078-5_21","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,9,30]]},"assertion":[{"value":"30 September 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Porto","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 September 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"19 September 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecmlpkdd.org\/2025\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}