{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,7]],"date-time":"2026-10-07T08:03:27Z","timestamp":1791360207877,"version":"4.3.1"},"reference-count":40,"publisher":"Association for Computing Machinery (ACM)","issue":"6","license":[{"start":{"date-parts":[[2023,5,24]],"date-time":"2023-05-24T00:00:00Z","timestamp":1684886400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2023,6]]},"abstract":"<jats:p>Examining the implicit bias in training neural networks using gradient-based methods.<\/jats:p>","DOI":"10.1145\/3571070","type":"journal-article","created":{"date-parts":[[2023,5,24]],"date-time":"2023-05-24T13:25:31Z","timestamp":1684934731000},"page":"86-93","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":42,"title":["On the Implicit Bias in Deep-Learning Algorithms"],"prefix":"10.1145","volume":"66","author":[{"given":"Gal","family":"Vardi","sequence":"first","affiliation":[{"name":"TTI-Chicago and the Hebrew University, Jerusalem, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,5,24]]},"reference":[{"key":"e_1_2_2_1_1","volume-title":"Advances in Neural Information Processing Systems","author":"Arora S.","year":"2019","unstructured":"Arora, S., Cohen, N., Hu, W., Luo, Y. Implicit regularization in deep matrix factorization. In Advances in Neural Information Processing Systems (2019), Curran Associates, Inc, Red Hook, NY, 7413--7424."},{"key":"e_1_2_2_2_1","volume-title":"Understanding gradient descent on edge of stability in deep learning. arXiv preprint arXiv:2205.09745","author":"Arora S.","year":"2022","unstructured":"Arora, S., Li, Z., Panigrahi, A. Understanding gradient descent on edge of stability in deep learning. arXiv preprint arXiv:2205.09745 (2022)."},{"key":"e_1_2_2_3_1","volume-title":"International Conference on Learning Representations (ICLR)","author":"Azizan N.","year":"2019","unstructured":"Azizan, N., Hassibi, B. Stochastic gradient\/mirror descent: Minimax optimality and implicit regularization. In International Conference on Learning Representations (ICLR) (2019), OpenReview.net."},{"key":"e_1_2_2_4_1","volume-title":"International Conference on Machine Learning","author":"Azulay S.","year":"2021","unstructured":"Azulay, S., Moroshko, E., Nacson, M.S., Woodworth, B., Srebro, N., Globerson, A., et al. On the implicit bias of initialization shape: Beyond infinitesimal mirror descent. In International Conference on Machine Learning (2021), PMLR, 468--477."},{"key":"e_1_2_2_5_1","first-page":"6240","article-title":"Spectrally-normalized margin bounds for neural networks","volume":"30","author":"Bartlett P.L.","year":"2017","unstructured":"Bartlett, P.L., Foster, D.J., Telgarsky, M.J. Spectrally-normalized margin bounds for neural networks. Adv. Neural Inf. Process. Syst. 30 (2017), 6240--6249.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1903070116"},{"key":"e_1_2_2_7_1","volume-title":"On gradient descent convergence beyond the edge of stability. arXiv preprint arXiv:2206.04172","author":"Chen L.","year":"2022","unstructured":"Chen, L., Bruna, J. On gradient descent convergence beyond the edge of stability. arXiv preprint arXiv:2206.04172 (2022)."},{"key":"e_1_2_2_8_1","volume-title":"Conference on Learning Theory","author":"Chizat L.","year":"2020","unstructured":"Chizat, L., Bach, F. Implicit bias of gradient descent for wide two-layer neural networks trained with the logistic loss. In Conference on Learning Theory (2020), PMLR, 1305--1338."},{"key":"e_1_2_2_9_1","volume-title":"International Conference on Learning Representations (ICLR)","author":"Cohen J.M.","year":"2021","unstructured":"Cohen, J.M., Kaur, S., Li, Y., Zico Kolter, J., Talwalkar, A. Gradient descent on neural networks typically occurs at the edge of stability. In International Conference on Learning Representations (ICLR) (2021), OpenReview.net."},{"key":"e_1_2_2_10_1","volume-title":"International Conference on Machine Learning","author":"Dinh L.","year":"2017","unstructured":"Dinh, L., Pascanu, R., Bengio, S., Bengio, Y. Sharp minima can generalize for deep nets. In International Conference on Machine Learning (2017), PMLR, 1019--1028."},{"key":"e_1_2_2_11_1","volume-title":"Conference On Learning Theory","author":"Golowich N.","year":"2018","unstructured":"Golowich, N., Rakhlin, A., Shamir, O. Size-independent sample complexity of neural networks. In Conference On Learning Theory (2018), PMLR, 297--299."},{"key":"e_1_2_2_12_1","volume-title":"International Conference on Machine Learning (2018","author":"Gunasekar S.","year":"1832","unstructured":"Gunasekar, S., Lee, J., Soudry, D., Srebro, N. Characterizing implicit bias in terms of optimization geometry. In International Conference on Machine Learning (2018), PMLR, 1832--1841."},{"key":"e_1_2_2_13_1","volume-title":"Advances in Neural Information Processing Systems","author":"Gunasekar S.","year":"2018","unstructured":"Gunasekar, S., Lee, J.D., Soudry, D., Srebro, N. Implicit bias of gradient descent on linear convolutional networks. In Advances in Neural Information Processing Systems (2018), Curran Associates, Inc, Red Hook, NY, 9461--9471."},{"key":"e_1_2_2_14_1","first-page":"6151","article-title":"Implicit regularization in matrix factorization","volume":"30","author":"Gunasekar S.","year":"2017","unstructured":"Gunasekar, S., Woodworth, E.B., Bhojanapalli, S., Neyshabur, B., Srebro, N. Implicit regularization in matrix factorization. Adv. Neural Inf. Process. Syst. 30 (2017), 6151--6159.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_15_1","volume-title":"Three factors influencing minima in SGD. arXiv preprint arXiv:1711.04623","author":"Jastrz\u0119bski S.","year":"2017","unstructured":"Jastrz\u0119bski, S., Kenton, Z., Arpit, D., Ballas, N., Fischer, A., Bengio, Y., et al. Three factors influencing minima in SGD. arXiv preprint arXiv:1711.04623 (2017)."},{"key":"e_1_2_2_16_1","volume-title":"International Conference on Learning Representations","author":"Ji Z.","year":"2018","unstructured":"Ji, Z., Telgarsky, M. Gradient descent aligns the layers of deep linear networks. In International Conference on Learning Representations (2018), OpenReview.net."},{"key":"e_1_2_2_17_1","volume-title":"Advances in Neural Information Processing Systems","author":"Ji Z.","year":"2020","unstructured":"Ji, Z., Telgarsky, M. Directional convergence and alignment in deep learning. In Advances in Neural Information Processing Systems (2020), Curran Associates, Inc, Red Hook, NY."},{"key":"e_1_2_2_18_1","volume-title":"International Conference on Learning Representations (ICLR)","author":"Keskar N.S.","year":"2017","unstructured":"Keskar, N.S., Mudigere, D., Nocedal, J., Smelyanskiy, M., Tang, P.T.P. On large-batch training for deep learning: Generalization gap and sharp minima. In International Conference on Learning Representations (ICLR) (2017), OpenReview.net."},{"key":"e_1_2_2_19_1","volume-title":"Conference On Learning Theory","author":"Li Y.","year":"2018","unstructured":"Li, Y., Ma, T., Zhang, H. Algorithmic regularization in over-parameterized matrix sensing and neural networks with quadratic activations. In Conference On Learning Theory (2018), PMLR, 2--47."},{"key":"e_1_2_2_20_1","volume-title":"K. Towards Resolving the Implicit Bias of Gradient Descent for Matrix Factorization: Greedy Low-Rank Learning. In International Conference on Learning Representations","author":"Li Z.","year":"2020","unstructured":"Li, Z., Luo, Y., Lyu, K. Towards Resolving the Implicit Bias of Gradient Descent for Matrix Factorization: Greedy Low-Rank Learning. In International Conference on Learning Representations (2020), OpenReview.net."},{"key":"e_1_2_2_21_1","volume-title":"International Conference on Learning Representations","author":"Lyu K.","year":"2019","unstructured":"Lyu, K., Li, J. Gradient descent maximizes the margin of homogeneous neural networks. In International Conference on Learning Representations (2019)."},{"key":"e_1_2_2_22_1","first-page":"12978","article-title":"Gradient descent on two-layer nets: Margin maximization and simplicity bias","volume":"34","author":"Lyu K.","year":"2021","unstructured":"Lyu, K., Li, Z., Wang, R., Arora, S. Gradient descent on two-layer nets: Margin maximization and simplicity bias. Adv. Neural Inf. Process. Syst. 34 (2021), 12978--12991.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_23_1","first-page":"16805","article-title":"On linear stability of SGD and input-smoothness of neural networks","volume":"34","author":"Ma C.","year":"2021","unstructured":"Ma, C., Ying, L. On linear stability of SGD and input-smoothness of neural networks. Adv. Neural Inf. Process. Syst. 34 (2021), 16805--16817.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_24_1","volume-title":"International Conference on Machine Learning","author":"Mulayoff R.","year":"2020","unstructured":"Mulayoff, R., Michaeli, T. Unique properties of flat minima in deep networks. In International Conference on Machine Learning (2020), PMLR, 7108--7118."},{"key":"e_1_2_2_25_1","first-page":"17749","article-title":"The implicit bias of minima stability: A view from function space","volume":"34","author":"Mulayoff R.","year":"2021","unstructured":"Mulayoff, R., Michaeli, T., Soudry, D. The implicit bias of minima stability: A view from function space. Adv. Neural Inf. Process. Syst. 34 (2021), 17749--17761.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_26_1","volume-title":"International Conference on Machine Learning","author":"Nacson M.S.","year":"2019","unstructured":"Nacson, M.S., Gunasekar, S., Lee, J., Srebro, N., Soudry, D. Lexicographic and depth-sensitive margins in homogeneous and non-homogeneous deep models. In International Conference on Machine Learning (2019), PMLR, 4683--4692."},{"key":"e_1_2_2_27_1","volume-title":"D. Implicit Bias of the Step Size in Linear Diagonal Neural Networks. In International Conference on Machine Learning","author":"Nacson M.S.","year":"2022","unstructured":"Nacson, M.S., Ravichandran, K., Srebro, N., Soudry, D. Implicit Bias of the Step Size in Linear Diagonal Neural Networks. In International Conference on Machine Learning (2022), PMLR, 16270--16295."},{"key":"e_1_2_2_28_1","volume-title":"Advances in Neural Information Processing Systems","author":"Neyshabur B.","year":"2017","unstructured":"Neyshabur, B., Bhojanapalli, S., McAllester, D., Srebro, N. Exploring generalization in deep learning. In Advances in Neural Information Processing Systems (2017), Curran Associates, Inc, Red Hook, NY, 5947--5956."},{"key":"e_1_2_2_29_1","first-page":"21174","article-title":"Implicit regularization in deep learning may not be explainable by norms","volume":"33","author":"Razin N.","year":"2020","unstructured":"Razin, N., Cohen, N. Implicit regularization in deep learning may not be explainable by norms. Adv. Neural Inf. Process. Syst. 33 (2020), 21174--21187.","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_30_1","volume-title":"On the effective number of linear regions in shallow univariate ReLU networks: Convergence guarantees and implicit bias. arXiv preprint arXiv:2205.09072","author":"Safran I.","year":"2022","unstructured":"Safran, I., Vardi, G., Lee, J.D. On the effective number of linear regions in shallow univariate ReLU networks: Convergence guarantees and implicit bias. arXiv preprint arXiv:2205.09072 (2022)."},{"key":"e_1_2_2_31_1","volume-title":"International Conference on Machine Learning","author":"Sarussi R.","year":"2021","unstructured":"Sarussi, R., Brutzkus, A., Globerson, A. Towards understanding learning in neural networks with linear teachers. In International Conference on Machine Learning (2021), PMLR, 9313--9322."},{"key":"e_1_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781107298019"},{"key":"e_1_2_2_33_1","first-page":"1","article-title":"The implicit bias of gradient descent on separable data","volume":"19","author":"Soudry D.","year":"2018","unstructured":"Soudry, D., Hoffer, E., Nacson, M.S., Gunasekar, S., Srebro, N. The implicit bias of gradient descent on separable data. J. Mach. Learn. Res. 19, 1 (2018), 2822--2878.","journal-title":"J. Mach. Learn. Res."},{"key":"e_1_2_2_34_1","volume-title":"Conference on Learning Theory","author":"Vardi G.","year":"2021","unstructured":"Vardi, G., Shamir, O. Implicit regularization in relu networks with the square loss. In Conference on Learning Theory (2021), PMLR, 4224--4258."},{"key":"e_1_2_2_35_1","volume-title":"On margin maximization in linear and ReLU networks. arXiv preprint arXiv:2110.02732","author":"Vardi G.","year":"2021","unstructured":"Vardi, G., Shamir, O., Srebro, N. On margin maximization in linear and ReLU networks. arXiv preprint arXiv:2110.02732 (2021)."},{"key":"e_1_2_2_36_1","volume-title":"Conference on Learning Theory","author":"Woodworth B.","year":"2020","unstructured":"Woodworth, B., Gunasekar, S., Lee, J.D., Moroshko, E., Savarese, P., Golan, I., et al. Kernel and rich regimes in overparametrized models. In Conference on Learning Theory (2020), PMLR, 3635--3673."},{"key":"e_1_2_2_37_1","first-page":"31","article-title":"How sgd selects the global minima in overparameterized learning: A dynamical stability perspective","author":"Wu L.","year":"2018","unstructured":"Wu, L., Ma, C., et al. How sgd selects the global minima in overparameterized learning: A dynamical stability perspective. Adv. Neural Inf. Process. Syst. 31 (2018).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"e_1_2_2_38_1","volume-title":"When does SGD favor flat minima? A quantitative characterization via linear stability. arXiv preprint arXiv:2207.02628","author":"Wu L.","year":"2022","unstructured":"Wu, L., Wang, M., Su, W. When does SGD favor flat minima? A quantitative characterization via linear stability. arXiv preprint arXiv:2207.02628 (2022)."},{"key":"e_1_2_2_39_1","volume-title":"International Conference on Learning Representations (ICLR)","author":"Yun C.","year":"2021","unstructured":"Yun, C., Krishnan, S., Mobahi, H. A unifying view on implicit bias in training linear neural networks. In International Conference on Learning Representations (ICLR) (2021), OpenReview.net."},{"key":"e_1_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3446776"}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3571070","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3571070","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:08:20Z","timestamp":1750183700000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3571070"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,5,24]]},"references-count":40,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2023,6]]}},"alternative-id":["10.1145\/3571070"],"URL":"https:\/\/doi.org\/10.1145\/3571070","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,5,24]]},"assertion":[{"value":"2023-05-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}