{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:46:08Z","timestamp":1740123968095,"version":"3.37.3"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2023,12,20]],"date-time":"2023-12-20T00:00:00Z","timestamp":1703030400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,12,20]],"date-time":"2023-12-20T00:00:00Z","timestamp":1703030400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100010669","name":"H2020 LEIT Information and Communication Technologies","doi-asserted-by":"publisher","award":["826452"],"award-info":[{"award-number":["826452"]}],"id":[{"id":"10.13039\/100010669","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Optim Theory Appl"],"published-print":{"date-parts":[[2024,2]]},"DOI":"10.1007\/s10957-023-02345-7","type":"journal-article","created":{"date-parts":[[2023,12,20]],"date-time":"2023-12-20T12:02:29Z","timestamp":1703073749000},"page":"634-668","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["On Stochastic Roundoff Errors in Gradient Descent with Low-Precision Computation"],"prefix":"10.1007","volume":"200","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4941-8999","authenticated-orcid":false,"given":"Lu","family":"Xia","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefano","family":"Massei","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michiel E.","family":"Hochstenbach","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Barry","family":"Koren","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,12,20]]},"reference":[{"issue":"3","key":"2345_CR1","doi-asserted-by":"publisher","first-page":"627","DOI":"10.1137\/S1052623497331063","volume":"10","author":"DP Bertsekas","year":"2000","unstructured":"Bertsekas, D.P., Tsitsiklis, J.N.: Gradient convergence in gradient methods with errors. SIAM J. Optim. 10(3), 627\u2013642 (2000)","journal-title":"SIAM J. Optim."},{"issue":"1","key":"2345_CR2","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1007\/BF00048682","volume":"44","author":"D B\u00f6hning","year":"1992","unstructured":"B\u00f6hning, D.: Multinomial logistic regression algorithm. Ann. Inst. Stat. Math. 44(1), 197\u2013200 (1992)","journal-title":"Ann. Inst. Stat. Math."},{"key":"2345_CR3","doi-asserted-by":"crossref","unstructured":"Chung, E., Fowers, J., Ovtcharov, K., Papamichael, M., Caulfield, A., Massengill, T., Liu, M., Lo, D., Alkalay, S., Haselman, M., Abeydeera, M., Adams, L., Angepat, H., Boehn, C., Chiou, D., Firestein, O., Forin, A., Gatlin, K., Ghandi, M., Heil, S., Holohan, K., Husseini, A., Juh\u00e1sz, T., Kagi, K., Kovvuri, R., Lanka, S., Megen, F.V., Mukhortov, D., Patel, P., Perez, B., Rapsang, A., Reinhardt, S., Rouhani, B., Sapek, A., Seera, R., Shekar, S., Sridharan, B., Weisz, G., Woods, L., Xiao, P.Y., Zhang, D., Zhao, R., Burger, D.: Serving DNNs in real time at datacenter scale with project brainwave. IEEE Micro 38(2), 8\u201320 (2018)","DOI":"10.1109\/MM.2018.022071131"},{"issue":"1","key":"2345_CR4","doi-asserted-by":"publisher","first-page":"A566","DOI":"10.1137\/20M1334796","volume":"43","author":"MP Connolly","year":"2021","unstructured":"Connolly, M.P., Higham, N.J., Mary, T.: Stochastic rounding and its probabilistic backward error analysis. SIAM J. Sci. Comput. 43(1), A566\u2013A585 (2021)","journal-title":"SIAM J. Sci. Comput."},{"issue":"3","key":"2345_CR5","doi-asserted-by":"publisher","DOI":"10.1098\/rsos.211631","volume":"9","author":"M Croci","year":"2022","unstructured":"Croci, M., Fasi, M., Higham, N.J., Mary, T., Mikaitis, M.: Stochastic rounding: implementation, error analysis and applications. R. Soc. Open Sci. 9(3), 211631 (2022)","journal-title":"R. Soc. Open Sci."},{"issue":"3","key":"2345_CR6","doi-asserted-by":"publisher","first-page":"1358","DOI":"10.1093\/imanum\/drac012","volume":"43","author":"M Croci","year":"2023","unstructured":"Croci, M., Giles, M.B.: Effects of round-to-nearest and stochastic rounding in the numerical solution of the heat equation in low precision. IMA J. Numer. Anal. 43(3), 1358\u20131390 (2023)","journal-title":"IMA J. Numer. Anal."},{"issue":"1","key":"2345_CR7","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1109\/MM.2018.112130359","volume":"38","author":"M Davies","year":"2018","unstructured":"Davies, M., Srinivasa, N., Lin, T.H., Chinya, G., Cao, Y., Choday, S.H., Dimou, G., Joshi, P., Imam, N., Jain, S., Liao, Y., Lin, C.-K., Lines, A., Liu, R., Mathaikutty, D., McCoy, S., Paul, A., Tse, J., Venkataramanan, G., Weng, Y.-H., Wild, A., Yang, Y., Wang, H.: Loihi: A neuromorphic manycore processor with on-chip learning. IEEE Micro 38(1), 82\u201399 (2018)","journal-title":"IEEE Micro"},{"issue":"6","key":"2345_CR8","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1109\/MSP.2012.2211477","volume":"29","author":"L Deng","year":"2012","unstructured":"Deng, L.: The MNIST database of handwritten digit images for machine learning research. IEEE Signal Process. Mag. 29(6), 141\u2013142 (2012)","journal-title":"IEEE Signal Process. Mag."},{"key":"2345_CR9","unstructured":"Glorot, X., Bengio, Y.: Understanding the difficulty of training deep feedforward neural networks. In: Proc. of the 13th Int. Conf. Artif. Intell. Stat., pp. 249\u2013256 (2010)"},{"key":"2345_CR10","unstructured":"Gupta, S., Agrawal, A., Gopalakrishnan, K., Narayanan, P.: Deep learning with limited numerical precision. In: Proc. of the 32nd Int. Conf. Mach. Learn., pp. 1737\u20131746 (2015)"},{"key":"2345_CR11","doi-asserted-by":"crossref","unstructured":"Hickmann, B., Chen, J., Rotzin, M., Yang, A., Urbanski, M., Avancha, S.: Intel Nervana neural network processor-t (NNP-T) fused floating point many-term dot product. In: Proc. of the 27th IEEE Symp. Comput., pp. 133\u2013136. IEEE (2020)","DOI":"10.1109\/ARITH48897.2020.00029"},{"key":"2345_CR12","doi-asserted-by":"publisher","DOI":"10.1137\/1.9780898718027","volume-title":"Accuracy and Stability of Numerical Algorithms","author":"NJ Higham","year":"2002","unstructured":"Higham, N.J.: Accuracy and Stability of Numerical Algorithms. SIAM, Philadelphia (2002)"},{"issue":"5","key":"2345_CR13","doi-asserted-by":"publisher","first-page":"C585","DOI":"10.1137\/19M1251308","volume":"41","author":"NJ Higham","year":"2019","unstructured":"Higham, N.J., Pranesh, S.: Simulating low precision floating-point arithmetic. SIAM J. Sci. Comput. 41(5), C585\u2013C602 (2019)","journal-title":"SIAM J. Sci. Comput."},{"issue":"2166","key":"2345_CR14","doi-asserted-by":"publisher","first-page":"20190052","DOI":"10.1098\/rsta.2019.0052","volume":"378","author":"M Hopkins","year":"2020","unstructured":"Hopkins, M., Mikaitis, M., Lester, D.R., Furber, S.: Stochastic rounding and reduced-precision fixed-point arithmetic for solving neural ordinary differential equations. Philos. Trans. Royal Soc. A 378(2166), 20190052 (2020)","journal-title":"Philos. Trans. Royal Soc. A"},{"key":"2345_CR15","doi-asserted-by":"publisher","DOI":"10.1002\/9781118548387","volume-title":"Applied Logistic Regression","author":"DW Hosmer Jr","year":"2013","unstructured":"Hosmer, D.W., Jr., Lemeshow, S., Sturdivant, R.X.: Applied Logistic Regression. John Wiley & Sons, Hoboken (2013)"},{"key":"2345_CR16","doi-asserted-by":"publisher","first-page":"57","DOI":"10.6028\/jres.042.005","volume":"42","author":"HD Huskey","year":"1949","unstructured":"Huskey, H.D., Hartree, D.R.: On the precision of a certain procedure of numerical integration. J. Res. Natl. Inst. Stand. Technol. 42, 57\u201362 (1949)","journal-title":"J. Res. Natl. Inst. Stand. Technol."},{"key":"2345_CR17","unstructured":"IEEE: IEEE standard for floating-point arithmetic. IEEE Std 754-2019 (Revision of IEEE 754-2008) pp. 1-84 (2019)"},{"issue":"7","key":"2345_CR18","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1145\/3360307","volume":"63","author":"NP Jouppi","year":"2020","unstructured":"Jouppi, N.P., Yoon, D.H., Kurian, G., Li, S., Patil, N., Laudon, J., Young, C., Patterson, D.: A domain-specific supercomputer for training deep neural networks. Commun. ACM 63(7), 67\u201378 (2020)","journal-title":"Commun. ACM"},{"key":"2345_CR19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-7643-8749-5","volume-title":"An Introduction to the Theory of Functional Equations and Inequalities: Cauchy\u2019s Equation and Jensen\u2019s Inequality","author":"M Kuczma","year":"2009","unstructured":"Kuczma, M.: An Introduction to the Theory of Functional Equations and Inequalities: Cauchy\u2019s Equation and Jensen\u2019s Inequality. Birkh\u00e4user Verlag AG, Basel (2009)"},{"key":"2345_CR20","unstructured":"Lee, J.D., Simchowitz, M., Jordan, M.I., Recht, B.: Gradient descent only converges to minimizers. In: Proc. of the 29th Annual Conf. on Learn. Theory, pp. 1246\u20131257 (2016)"},{"key":"2345_CR21","unstructured":"Li, H., De, S., Xu, Z., Studer, C., Samet, H., Goldstein, T.: Training quantized nets: A deeper understanding. In: Proc. of the 31st Neural Inf. Process. Syst. Conf., vol.\u00a030 (2017)"},{"issue":"1","key":"2345_CR22","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1109\/TFUZZ.2015.2418000","volume":"24","author":"Y Liu","year":"2015","unstructured":"Liu, Y., Gao, Y., Tong, S., Li, Y.: Fuzzy approximation-based adaptive backstepping optimal control for a class of nonlinear discrete-time systems with dead-zone. IEEE Trans. Fuzzy Syst. 24(1), 16\u201328 (2015)","journal-title":"IEEE Trans. Fuzzy Syst."},{"key":"2345_CR23","doi-asserted-by":"crossref","unstructured":"Mikaitis, M.: Stochastic rounding: Algorithms and hardware accelerator. In: Proc. of 2021 Int. Jt. Conf. Neural Netw., pp. 1\u20136. IEEE (2021)","DOI":"10.1109\/IJCNN52387.2021.9533756"},{"key":"2345_CR24","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1016\/j.ins.2019.04.012","volume":"492","author":"E Moulay","year":"2019","unstructured":"Moulay, E., L\u00e9chapp\u00e9, V., Plestan, F.: Properties of the sign gradient descent algorithms. Inf. Sci. 492, 29\u201339 (2019)","journal-title":"Inf. Sci."},{"key":"2345_CR25","doi-asserted-by":"crossref","unstructured":"Na, T., Ko, J.H., Kung, J., Mukhopadhyay, S.: On-chip training of recurrent neural networks with limited numerical precision. In: Proc. of the 2017 Int. Jt. Conf. Neural Netw., pp. 3716\u20133723. IEEE (2017)","DOI":"10.1109\/IJCNN.2017.7966324"},{"key":"2345_CR26","volume-title":"Introductory Lectures on Convex Optimization: A Basic Course","author":"Y Nesterov","year":"2003","unstructured":"Nesterov, Y.: Introductory Lectures on Convex Optimization: A Basic Course. Springer, New York (2003)"},{"key":"2345_CR27","unstructured":"NVIDIA H100 tensor core GPU architecture [white paper] (2022)"},{"key":"2345_CR28","unstructured":"Ortiz, M., Cristal, A., Ayguad\u00e9, E., Casas, M.: Low-precision floating-point schemes for neural network training. arXiv preprint: 1804.05267 (2018)"},{"issue":"4","key":"2345_CR29","doi-asserted-by":"publisher","first-page":"1215","DOI":"10.1175\/JCLI-D-21-0343.1","volume":"35","author":"EA Paxton","year":"2022","unstructured":"Paxton, E.A., Chantry, M., Kl\u00f6wer, M., Saffin, L., Palmer, T.: Climate modeling in low precision: Effects of both deterministic and stochastic rounding. J. Clim. 35(4), 1215\u20131229 (2022)","journal-title":"J. Clim."},{"issue":"2","key":"2345_CR30","doi-asserted-by":"publisher","first-page":"331","DOI":"10.1109\/TRO.2007.895057","volume":"23","author":"C Petres","year":"2007","unstructured":"Petres, C., Pailhas, Y., Patron, P., Petillot, Y., Evans, J., Lane, D.: Path planning for autonomous underwater vehicles. IEEE Trans. Robot. 23(2), 331\u2013341 (2007)","journal-title":"IEEE Trans. Robot."},{"key":"2345_CR31","unstructured":"Schmidt, M., Roux, N., Bach, F.: Convergence rates of inexact proximal-gradient methods for convex optimization. In: Proc. of the 24th Neural Inf. Process. Syst. Conf., pp. 1458\u20131466 (2011)"},{"key":"2345_CR32","first-page":"1331","volume":"57","author":"H Singh","year":"1988","unstructured":"Singh, H., Upadhyaya, L., Namjoshi, U.: Estimation of finite population variance. Curr. Sci. 57, 1331\u20131334 (1988)","journal-title":"Curr. Sci."},{"key":"2345_CR33","volume-title":"Probability and Conditional Expectation: Fundamentals for the Empirical Sciences","author":"R Steyer","year":"2017","unstructured":"Steyer, R., Nagel, W.: Probability and Conditional Expectation: Fundamentals for the Empirical Sciences. John Wiley & Sons, Oxford (2017)"},{"issue":"2","key":"2345_CR34","doi-asserted-by":"publisher","DOI":"10.1088\/1674-4926\/41\/2\/022404","volume":"41","author":"C Su","year":"2020","unstructured":"Su, C., Zhou, S., Feng, L., Zhang, W.: Towards high performance low bitwidth training for deep neural networks. J. Semicond. 41(2), 022404 (2020)","journal-title":"J. Semicond."},{"key":"2345_CR35","unstructured":"Wang, N., Choi, J., Brand, D., Chen, C.Y., Gopalakrishnan, K.: Training deep neural networks with 8-bit floating point numbers. In: Proc. of the 31st Neural Inf. Process. Syst. Conf., pp. 7675\u20137684 (2018)"},{"issue":"3","key":"2345_CR36","doi-asserted-by":"publisher","first-page":"467","DOI":"10.1007\/s10994-019-05839-6","volume":"109","author":"D Zou","year":"2020","unstructured":"Zou, D., Cao, Y., Zhou, D., Gu, Q.: Gradient descent optimizes over-parameterized deep ReLU networks. Mach. Learn. 109(3), 467\u2013492 (2020)","journal-title":"Mach. Learn."}],"container-title":["Journal of Optimization Theory and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10957-023-02345-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10957-023-02345-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10957-023-02345-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,31]],"date-time":"2024-01-31T13:16:10Z","timestamp":1706706970000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10957-023-02345-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,20]]},"references-count":36,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024,2]]}},"alternative-id":["2345"],"URL":"https:\/\/doi.org\/10.1007\/s10957-023-02345-7","relation":{},"ISSN":["0022-3239","1573-2878"],"issn-type":[{"type":"print","value":"0022-3239"},{"type":"electronic","value":"1573-2878"}],"subject":[],"published":{"date-parts":[[2023,12,20]]},"assertion":[{"value":"24 February 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 November 2023","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 December 2023","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}