{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T18:42:38Z","timestamp":1767292958541,"version":"3.48.0"},"reference-count":39,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Statistical Inference Generates Knowledge for Artificial Learners"},{"DOI":"10.13039\/100011661","name":"Pacific Northwest National Laboratory","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100011661","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Mathematics for Artificial Reasoning in Science"},{"DOI":"10.13039\/100007000","name":"Laboratory Directed Research and Development","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007000","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Program, PNNL"},{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Battelle Memorial Institute under Contract","award":["DE-AC05-76RL0-1830"],"award-info":[{"award-number":["DE-AC05-76RL0-1830"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE J. Sel. Top. Signal Process."],"published-print":{"date-parts":[[2025,10]]},"DOI":"10.1109\/jstsp.2025.3583140","type":"journal-article","created":{"date-parts":[[2025,6,27]],"date-time":"2025-06-27T13:46:05Z","timestamp":1751031965000},"page":"1477-1492","source":"Crossref","is-referenced-by-count":0,"title":["Measuring Training Variability From Stochastic Optimization Using Robust Nonparametric Testing"],"prefix":"10.1109","volume":"19","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1028-4863","authenticated-orcid":false,"given":"Sinjini","family":"Banerjee","sequence":"first","affiliation":[{"name":"Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5240-3053","authenticated-orcid":false,"given":"Tim","family":"Marrinan","sequence":"additional","affiliation":[{"name":"Pacific Northwest National Lab, Richland, WA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8227-558X","authenticated-orcid":false,"given":"Reilly","family":"Cannon","sequence":"additional","affiliation":[{"name":"Pacific Northwest National Lab, Richland, WA, USA"}]},{"given":"Tony","family":"Chiang","sequence":"additional","affiliation":[{"name":"Pacific Northwest National Laboratory and ARPA-H, Bethesda, MD, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6123-5282","authenticated-orcid":false,"given":"Anand D.","family":"Sarwate","sequence":"additional","affiliation":[{"name":"Department of Electrical and Computer Engineering, Rutgers, The State University of New Jersey, Piscataway, NJ, USA"}]}],"member":"263","reference":[{"article-title":"Robust nonparametric hypothesis testing to understand variability in training neural networks","year":"2023","author":"Banerjee","key":"ref1"},{"article-title":"Measuring model variability using robust non-parametric testing","year":"2024","author":"Banerjee","key":"ref2"},{"article-title":"Sources of irreproducibility in machine learning: A review","year":"2023","author":"Gundersen","key":"ref3"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"ref5","first-page":"338","article-title":"Reporting score distributions makes a difference: Performance study of LSTM-networks for sequence tagging","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Reimers","year":"2017"},{"article-title":"On the state of the art of evaluation in neural language models","year":"2017","author":"Melis","key":"ref6"},{"key":"ref7","first-page":"725","article-title":"Unreproducible research is reproducible","author":"Bouthillier","year":"2019"},{"article-title":"Torch.manual_seed (3407) is all you need: On the influence of random seeds in deep learning architectures for computer vision","year":"2021","author":"Picard","key":"ref8"},{"article-title":"Deep ensembles: A loss landscape perspective","year":"2019","author":"Fort","key":"ref9"},{"key":"ref10","first-page":"747","article-title":"Accounting for variance in machine learning benchmarks","volume-title":"Proc. Mach. Learn. Syst.","author":"Bouthillier","year":"2021"},{"key":"ref11","first-page":"9913","article-title":"Nondeterminism and instability in neural network optimization","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","author":"Summers","year":"2021"},{"article-title":"Calibrated chaos: Variance between runs of neural network training is harmless and inevitable","year":"2023","author":"Jordan","key":"ref12"},{"article-title":"Fine-tuning pretrained language models: Weight initializations, data orders, and early stopping","year":"2020","author":"Dodge","key":"ref13"},{"key":"ref14","first-page":"3179","article-title":"Launch and iterate: Reducing prediction churn","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Fard","year":"2016"},{"key":"ref15","first-page":"642","article-title":"Asymptotic minimax character of the sample distribution function and of the classical multinomial estimator","volume-title":"Ann. Math. Statist.","volume":"27","author":"Dvoretzky","year":"1956"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.spl.2011.11.012"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1201\/9781315110479"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1002\/SERIES1345"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/0021-9045(91)90072-i"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1198\/016214508000000274"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1214\/09-aihp354"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1007\/s11749-019-00691-1"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v29i1.9602"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/1102351.1102430"},{"article-title":"Learning multiple layers of features from tiny images","year":"2009","author":"Krizhevsky","key":"ref25"},{"key":"ref26","article-title":"Amarel - office of advanced research computing"},{"author":"Banerjee","key":"ref27","article-title":"Two sample robust kolmogorov-smirnov test"},{"key":"ref28","first-page":"6405","article-title":"Simple and scalable predictive uncertainty estimation using deep ensembles","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Lakshminarayanan","year":"2017"},{"article-title":"Why m heads are better than one: Training a diverse ensemble of deep networks","year":"2015","author":"Lee","key":"ref29"},{"article-title":"Distilling the knowledge in a neural network","year":"2015","author":"Hinton","key":"ref30"},{"issue":"359","key":"ref31","first-page":"1","article-title":"A unified theory of diversity in ensemble learning","volume-title":"J. Mach. Learn. Res.","volume":"24","author":"Wood","year":"2023"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref33","first-page":"38","article-title":"Huggingfaces transformers: State-of-the-art natural language processing","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.: Syst. Demonstrations","author":"Wolf","year":"2020"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2015.02.042"},{"key":"ref35","article-title":"Pathologies of predictive diversity in deep ensembles","volume-title":"Trans. Mach. Learn. Res.","author":"Abe","year":"2024"},{"key":"ref36","first-page":"15015","article-title":"When are ensembles really effective?","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Theisen","year":"2023"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.3150\/11-bej351"},{"key":"ref38","first-page":"8580","article-title":"Neural tangent kernel: Convergence and generalization in neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Jacot","year":"2018"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/s10957-020-01743-5"}],"container-title":["IEEE Journal of Selected Topics in Signal Processing"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/4200690\/11320985\/11053660-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/4200690\/11320985\/11053660.pdf?arnumber=11053660","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T18:37:27Z","timestamp":1767292647000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11053660\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10]]},"references-count":39,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/jstsp.2025.3583140","relation":{},"ISSN":["1932-4553","1941-0484"],"issn-type":[{"type":"print","value":"1932-4553"},{"type":"electronic","value":"1941-0484"}],"subject":[],"published":{"date-parts":[[2025,10]]}}}