{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,21]],"date-time":"2026-08-21T17:25:03Z","timestamp":1787333103436,"version":"build-2736575974"},"reference-count":81,"publisher":"Society for Industrial & Applied Mathematics (SIAM)","issue":"1","funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-2208456"],"award-info":[{"award-number":["DMS-2208456"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-2208412"],"award-info":[{"award-number":["DMS-2208412"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["SIAM J. Matrix Anal. Appl."],"published-print":{"date-parts":[[2024,3,31]]},"abstract":"<jats:p>Abstract.<\/jats:p>\n                  <jats:p>This paper develops a new class of nonlinear acceleration algorithms based on extending conjugate residual-type procedures from linear to nonlinear equations. The main algorithm has strong similarities with Anderson acceleration as well as with inexact Newton methods\u2014depending on which variant is implemented. We prove theoretically and verify experimentally, on a variety of problems from simulation experiments to deep learning applications, that our method is a powerful accelerated iterative algorithm. The code is available at https:\/\/github.com\/Data-driven-numerical-methods\/Nonlinear-Truncated-Conjugate-Residual .<\/jats:p>","DOI":"10.1137\/23m1576360","type":"journal-article","created":{"date-parts":[[2024,2,29]],"date-time":"2024-02-29T05:48:20Z","timestamp":1709185700000},"page":"712-743","source":"Crossref","is-referenced-by-count":5,"title":["nlTGCR: A Class of Nonlinear Acceleration Procedures Based on Conjugate Residuals"],"prefix":"10.1137","volume":"45","author":[{"given":"Huan","family":"He","sequence":"first","affiliation":[{"name":"Department of Computer Science, Emory University, Atlanta, GA 30322 USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9594-6840","authenticated-orcid":true,"given":"Ziyuan","family":"Tang","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, University of Minnesota, Minneapolis, MN 55455 USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shifan","family":"Zhao","sequence":"additional","affiliation":[{"name":"Department of Mathematics, Emory University, Atlanta, GA 30322 USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yousef","family":"Saad","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, University of Minnesota, Minneapolis, MN 55455 USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0361-0931","authenticated-orcid":true,"given":"Yuanzhe","family":"Xi","sequence":"additional","affiliation":[{"name":"Department of Mathematics, Emory University, Atlanta, GA 30322 USA."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"351","published-online":{"date-parts":[[2024,2,29]]},"reference":[{"key":"ref1","unstructured":"PyTorch, https:\/\/www.pytorch.org."},{"key":"ref2","unstructured":"PyTorch-Geometric, https:\/\/www.pyg.org\/."},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/321296.321305"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.2140\/pjm.1966.16.1"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1016\/0024-3795(80)90226-8"},{"key":"ref6","doi-asserted-by":"crossref","unstructured":"W. Bian , \nX. Chen , and \nC. T. Kelley  , Anderson acceleration for a class of nonsmooth fixed-point problems, SIAM J. Sci. Comput., 43 (2021), pp. S1\u2013S20, https:\/\/doi.org\/10.1137\/20M132938X.","DOI":"10.1137\/20M132938X"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/s10589-019-00164-z"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s10589-020-00256-1"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1093\/imanum\/drab061"},{"key":"ref10","doi-asserted-by":"crossref","unstructured":"C. Brezinski  and \nM. Redivo-Zaglia  , The simplified topological \\(\\varepsilon\\)-algorithms for accelerating sequences in a vector space, SIAM J. Sci. Comput., 36 (2014), pp. A2227\u2013A2247, https:\/\/doi.org\/10.1137\/140957044.","DOI":"10.1137\/140957044"},{"key":"ref11","doi-asserted-by":"crossref","unstructured":"C. Brezinski , \nM. Redivo-Zaglia , and \nY. Saad  , Shanks sequence transformations and Anderson acceleration, SIAM Rev., 60 (2018), pp. 646\u2013669, https:\/\/doi.org\/10.1137\/17M1120725.","DOI":"10.1137\/17M1120725"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1137\/0911026"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1137\/0804017"},{"key":"ref14","unstructured":"A. Cabana  and \nL. F. Lago-Fern\u00e1ndez  , Backward Gradient Normalization in Deep Neural Networks, https:\/\/arxiv.org\/abs\/2106.09475, 2021."},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1137\/0713060"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-68648-6"},{"key":"ref17","unstructured":"R. T. Q. Chen , \nY. Rubanova , \nJ. Bettencourt , and \nD. Duvenaud  , Neural Ordinary Differential Equations, https:\/\/arxiv.org\/abs\/1806.07366, 2019."},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1051\/m2an\/2021069"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1561\/2400000036"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.compstruc.2008.11.013"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1137\/0719025"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1137\/0720023"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1137\/0804022"},{"key":"ref24","doi-asserted-by":"crossref","unstructured":"S. C. Eisenstat  and \nH. F. Walker  , Choosing the forcing terms in an inexact Newton method, SIAM J. Sci. Comput., 17 (1996), pp. 16\u201332, https:\/\/doi.org\/10.1137\/0917003.","DOI":"10.1137\/0917003"},{"key":"ref25","doi-asserted-by":"crossref","unstructured":"C. Evans , \nS. Pollock , \nL. G. Rebholz , and \nM. Xiao  , A proof that Anderson acceleration improves the convergence rate in linearly converging fixed-point methods (but not in those converging quadratically), SIAM J. Numer. Anal., 58 (2020), pp. 788\u2013810, https:\/\/doi.org\/10.1137\/19M1245384.","DOI":"10.1137\/19M1245384"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1006\/jcph.1996.0059"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2009.09.035"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/7.2.149"},{"key":"ref29","series-title":"Math. Notes 102","volume-title":"Introduction to Partial Differential Equations","author":"Folland G. B.","year":"1995"},{"key":"ref30","unstructured":"M. Geist  and \nB. Scherrer  , Anderson Acceleration for Reinforcement Learning, https:\/\/arxiv.org\/abs\/1809.09501, 2018."},{"key":"ref31","doi-asserted-by":"crossref","unstructured":"I. Griva , \nS. G. Nash , and \nA. Sofer  , Linear and Nonlinear Optimization, SIAM, Philadelphia, 2009.","DOI":"10.1137\/1.9780898717730"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/s11075-017-0455-1"},{"key":"ref33","doi-asserted-by":"crossref","unstructured":"H. He , \nY. Xi , and \nJ. C. Ho  , Fast and accurate tensor decomposition without a high performance computing machine, in Proceedings of the IEEE International Conference on Big Data (Big Data), IEEE, 2020, pp. 163\u2013170.","DOI":"10.1109\/BigData50022.2020.9378111"},{"key":"ref34","unstructured":"H. He , \nS. Zhao , \nZ. Tang , \nJ. C. Ho , \nY. Saad , and \nY. Xi  , An Efficient Nonlinear Acceleration Method that Exploits Symmetry of the Hessian, https:\/\/arxiv.org\/abs\/2210.12573, 2022."},{"key":"ref35","volume-title":"NeurIPS 2021 Workshop on Deep Generative Models and Downstream Applications","author":"He H.","year":"2021"},{"key":"ref36","unstructured":"H. He , \nS. Zhao , \nY. Xi , \nJ. Ho , and \nY. Saad  , GDA-AM: On the effectiveness of solving minimax optimization via Anderson mixing, in Proceedings of the International Conference on Learning Representations, 2022, https:\/\/openreview.net\/forum?id=3YqeuCVwy1d."},{"key":"ref37","unstructured":"K. He , \nX. Zhang , \nS. Ren , and \nJ. Sun  , Deep Residual Learning for Image Recognition, https:\/\/arxiv.org\/abs\/1512.03385, 2015."},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1093\/imanum\/19.4.549"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1016\/0024-3795(80)90165-2"},{"key":"ref40","unstructured":"H. Kasai  , hiroyuki-kasai\/GDLibrary, https:\/\/github.com\/hiroyuki-kasai\/GDLibrary."},{"key":"ref41","doi-asserted-by":"crossref","unstructured":"C. T. Kelley  , Iterative Methods for Linear and Nonlinear Equations, Frontiers in Appl. Math. 16, SIAM, Philadelphia, 1995.","DOI":"10.1137\/1.9781611970944"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1137\/1.9780898718898"},{"key":"ref43","unstructured":"D. P. Kingma  and \nJ. Ba  , Adam: A Method for Stochastic Optimization, https:\/\/arxiv.org\/abs\/1412.6980, 2017."},{"key":"ref44","unstructured":"T. N. Kipf  and \nM. Welling  , Semi-supervised Classification with Graph Convolutional Networks, preprint, https:\/\/arxiv.org\/abs\/1609.02907, 2016."},{"key":"ref45","unstructured":"A. Krizhevsky  , Learning Multiple Layers of Features from Tiny Images, 2009, https:\/\/www.cs.toronto.edu\/\u223ckriz\/learning-features-2009-TR.pdf."},{"key":"ref46","doi-asserted-by":"crossref","unstructured":"L. Lin  and \nC. Yang  , Elliptic preconditioner for accelerating the self-consistent field iteration in Kohn\u2013Sham density functional theory, SIAM J. Sci. Comput., 35 (2013), pp. S277\u2013S298, https:\/\/doi.org\/10.1137\/120880604.","DOI":"10.1137\/120880604"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/BF01589116"},{"key":"ref48","unstructured":"I. Loshchilov  and \nF. Hutter  , Decoupled Weight Decay Regularization, https:\/\/arxiv.org\/abs\/1711.05101, 2019."},{"key":"ref49","unstructured":"V. Mai  and \nM. Johansson  , Anderson acceleration of proximal gradient methods, in Proceedings of the 37th International Conference on Machine Learning, Proc. Mach. Learn. Res. 119,  \nH. D. III  and \nA. Singh  , eds. PMLR, 2020, pp. 6620\u20136629, http:\/\/proceedings.mlr.press\/v119\/mai20a.html."},{"key":"ref50","first-page":"4903","author":"Mai V. V.","year":"2019","journal-title":"Proceedings of the IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009953814988"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-55251-0"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1063\/1.1725600"},{"key":"ref54","volume-title":"Introductory Lectures on Convex Optimization: A Basic Course","author":"Nesterov Y.","year":"2014"},{"key":"ref55","doi-asserted-by":"crossref","unstructured":"C. W. Oosterlee  and \nT. Washio  , Krylov subspace acceleration of nonlinear multigrid with application to recirculating flows, SIAM J. Sci. Comput., 21 (2000), pp. 1670\u20131690, https:\/\/doi.org\/10.1137\/S1064827598338093.","DOI":"10.1137\/S1064827598338093"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/SoutheastCon48659.2022.9763953"},{"key":"ref57","doi-asserted-by":"crossref","unstructured":"S. Pollock , \nL. G. Rebholz , and \nM. Xiao  , Anderson-accelerated convergence of Picard iterations for incompressible Navier\u2013Stokes equations, SIAM J. Numer. Anal., 57 (2019), pp. 615\u2013637, https:\/\/doi.org\/10.1137\/18M1206151.","DOI":"10.1137\/18M1206151"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-12-597050-1.50006-3"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1016\/0009-2614(80)80396-4"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1002\/jcc.540030413"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1002\/nla.617"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729586"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1137\/1.9780898718003"},{"key":"ref64","doi-asserted-by":"crossref","unstructured":"Y. Saad  , Numerical Methods for Large Eigenvalue Problems, SIAM, Philadelphia, 2011, https:\/\/doi.org\/10.1137\/1.9781611970739.","DOI":"10.1137\/1.9781611970739"},{"key":"ref65","doi-asserted-by":"crossref","unstructured":"R. B. Schnabel  , Quasi-Newton Methods Using Multiple Secant Equations, Technical Report CU-CS-247-83, Department of Computer Science, University of Colorado at Boulder, 1983.","DOI":"10.21236\/ADA131444"},{"key":"ref66","unstructured":"D. Scieur , \nL. Liu , \nT. Pumir , and \nN. Boumal  , Generalization of quasi-Newton methods: Application to robust symmetric multisecant updates, in Proceedings of the 24th International Conference on Artificial Intelligence and Statistics, Proc. Mach. Learn. Res. 130,  \nA. Banerjee  and \nK. Fukumizu  , eds. PMLR, 2021, pp. 550\u2013558, https:\/\/proceedings.mlr.press\/v130\/scieur21a.html."},{"key":"ref67","unstructured":"D. Scieur , \nE. Oyallon , \nA. d\u2019Aspremont , and \nF. Bach  , Online Regularized Nonlinear Acceleration, https:\/\/arxiv.org\/abs\/1805.09639, 2018."},{"key":"ref68","volume-title":"NeurIPS","author":"Shi W.","year":"2019"},{"key":"ref69","doi-asserted-by":"crossref","unstructured":"D. A. Smith , \nW. F. Ford , and \nA. Sidi  , Extrapolation methods for vector sequences, SIAM Rev., 29 (1987), pp. 199\u2013233, https:\/\/doi.org\/10.1137\/1029042.","DOI":"10.1137\/1029042"},{"key":"ref70","unstructured":"S. Sreenivas , \nS Mandava , and \nC. Forster  , Pretraining BERT with Layer-wise Adaptive Learning Rates, 2019, https:\/\/developer.nvidia.com\/blog\/pretraining-bert-with-layer-wise-adaptive-learning-rates."},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/BF02564277"},{"key":"ref72","volume-title":"NeurIPS","author":"Sun K.","year":"2021"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1137\/130919398"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevB.30.6118"},{"key":"ref75","doi-asserted-by":"crossref","unstructured":"P. K. W. Vinsome  , ORTHOMIN: An iterative method for solving sparse sets of simultaneous linear equations, in Proceedings of the Fourth Symposium on Reservoir Simulation, Society of Petroleum Engineers, AIME, 1976, pp. 149\u2013159.","DOI":"10.2118\/5729-MS"},{"key":"ref76","doi-asserted-by":"crossref","unstructured":"H. F. Walker  and \nP. Ni  , Anderson acceleration for fixed-point iterations, SIAM J. Numer. Anal., 49 (2011), pp. 1715\u20131735, https:\/\/doi.org\/10.1137\/10078356X.","DOI":"10.1137\/10078356X"},{"key":"ref77","first-page":"22995","volume":"34","author":"Wei F.","year":"2021","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref78","unstructured":"F. Wei , \nC. Bao , and \nY. Liu  , A class of short-term recurrence Anderson mixing methods and their applications, in Proceedings of the International Conference on Learning Representations, 2022, https:\/\/openreview.net\/forum?id=_X90SIKbHa."},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511812545"},{"key":"ref80","doi-asserted-by":"crossref","unstructured":"Z. Yao , \nA. Gholami , \nS. Shen , \nK. Keutzer , and \nM. W. Mahoney  , Adahessian: An adaptive second order optimizer for machine learning, in Proceedings of the AAAI Conference on Artificial Intelligence, 2021.","DOI":"10.1609\/aaai.v35i12.17275"},{"key":"ref81","unstructured":"J. Zhang , \nB. O\u2019Donoghue , and \nS. Boyd  , Globally Convergent Type-I Anderson Acceleration for Non-smooth Fixed-point Iterations, https:\/\/arxiv.org\/abs\/1808.03971, 2018."}],"container-title":["SIAM Journal on Matrix Analysis and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/epubs.siam.org\/doi\/pdf\/10.1137\/23M1576360","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,21]],"date-time":"2026-08-21T16:32:28Z","timestamp":1787329948000},"score":1,"resource":{"primary":{"URL":"https:\/\/epubs.siam.org\/doi\/10.1137\/23M1576360"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2,29]]},"references-count":81,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2024,3,31]]}},"alternative-id":["10.1137\/23M1576360"],"URL":"https:\/\/doi.org\/10.1137\/23m1576360","relation":{},"ISSN":["0895-4798","1095-7162"],"issn-type":[{"value":"0895-4798","type":"print"},{"value":"1095-7162","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,2,29]]}}}