{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T19:23:36Z","timestamp":1740165816004,"version":"3.37.3"},"reference-count":73,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2020AAA0108104"],"award-info":[{"award-number":["2020AAA0108104"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1109\/tnnls.2022.3147031","type":"journal-article","created":{"date-parts":[[2022,2,14]],"date-time":"2022-02-14T23:00:01Z","timestamp":1644879601000},"page":"7910-7920","source":"Crossref","is-referenced-by-count":1,"title":["Towards Better Generalization of Deep Neural Networks via Non-Typicality Sampling Scheme"],"prefix":"10.1109","volume":"34","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7181-2159","authenticated-orcid":false,"given":"Xinyu","family":"Peng","sequence":"first","affiliation":[{"name":"Department of Automation, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9185-3989","authenticated-orcid":false,"given":"Fei-Yue","family":"Wang","sequence":"additional","affiliation":[{"name":"State Key Laboratory of Management and Control for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9428-1960","authenticated-orcid":false,"given":"Li","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Automation, Tsinghua National Laboratory for Information Science and Technology, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref13","article-title":"Large batch optimization for deep learning: Training BERT in 76 minutes","author":"you","year":"2019","journal-title":"arXiv 1904 00962"},{"key":"ref57","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","author":"keskar","year":"2016","journal-title":"arXiv 1609 04836"},{"key":"ref12","article-title":"Accurate, large minibatch SGD: Training ImageNet in 1 hour","author":"goyal","year":"2017","journal-title":"arXiv 1706 02677"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.1.1"},{"journal-title":"Sampling Design and Analysis","year":"2009","author":"lohr","key":"ref15"},{"key":"ref59","article-title":"PyHessian: Neural networks through the lens of the Hessian","author":"yao","year":"2019","journal-title":"arXiv 1912 07145"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729586"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1088\/1742-5468\/ab39d9"},{"key":"ref53","first-page":"6389","article-title":"Visualizing the loss landscape of neural nets","author":"li","year":"2018","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1137\/130908257"},{"key":"ref11","article-title":"World models","author":"ha","year":"2018","journal-title":"arXiv 1803 10122"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1080\/00268976.2018.1483535"},{"key":"ref10","first-page":"5998","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref54","article-title":"Empirical analysis of the Hessian of over-parametrized neural networks","author":"sagun","year":"2017","journal-title":"arXiv 1706 04454"},{"key":"ref17","article-title":"Are all training examples created equal? An empirical study","author":"vodrahalli","year":"2018","journal-title":"arXiv 1811 12569"},{"key":"ref16","article-title":"Not all samples are created equal: Deep learning with importance sampling","author":"katharopoulos","year":"2018","journal-title":"arXiv 1803 00942"},{"key":"ref19","first-page":"1","article-title":"Stochastic optimization with importance sampling for regularized loss minimization","author":"zhao","year":"2015","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.746"},{"key":"ref51","volume":"5","author":"bertsekas","year":"1996","journal-title":"Neuro-Dynamic Programming"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1080\/17442509408833885"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1080\/10556788.2017.1298596"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/s11590-015-0916-1"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46478-7_29"},{"key":"ref47","first-page":"865","article-title":"Quartz: Randomized dual coordinate ascent with arbitrary sampling","volume":"28","author":"qu","year":"2015","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/2393347.2396324"},{"key":"ref41","first-page":"2379","article-title":"Self-paced learning for long-term tracking","author":"supancic","year":"2013","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654918"},{"key":"ref43","first-page":"1002","article-title":"Active bias: Training more accurate neural networks by emphasizing high variance samples","author":"chang","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_12"},{"key":"ref8","first-page":"1","article-title":"Stardata: A starcraft ai research dataset","author":"lin","year":"2017","journal-title":"Proc 13th Artif Intell Interact Digit Entertainment Conf"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"ref4","first-page":"3104","article-title":"Sequence to sequence learning with neural networks","author":"sutskever","year":"2014","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"mnih","year":"2015","journal-title":"Nature"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2205597"},{"key":"ref5","article-title":"Seq2SQL: Generating structured queries from natural language using reinforcement learning","author":"zhong","year":"2017","journal-title":"arXiv 1709 00103"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2957003"},{"key":"ref35","first-page":"3621","article-title":"An empirical study of stochastic gradient descent with structured covariance noise","author":"wen","year":"2020","journal-title":"Proc Int Conf Artif Intell Statist"},{"key":"ref34","first-page":"10367","article-title":"On the noisy gradient descent that generalizes as SGD","author":"wu","year":"2020","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref37","first-page":"1143","article-title":"Control batch size and learning rate to generalize well: Theoretical and empirical evidence","author":"he","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref36","article-title":"The anisotropic noise in stochastic gradient descent: Its behavior of escaping from sharp minima and regularization effects","author":"zhu","year":"2018","journal-title":"arXiv 1803 00195"},{"key":"ref31","article-title":"Decoupling representation and classifier for long-tailed recognition","author":"kang","year":"2019","journal-title":"arXiv 1910 09217"},{"key":"ref30","first-page":"2304","article-title":"Mentornet: Learning data-driven curriculum for very deep neural networks on corrupted labels","author":"jiang","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00512"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00974"},{"key":"ref2","first-page":"1097","article-title":"ImageNet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref1","first-page":"91","article-title":"Faster R-CNN: Towards real-time object detection with region proposal networks","author":"ren","year":"2015","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref39","article-title":"Three factors influencing minima in SGD","author":"jastrz?bski","year":"2017","journal-title":"arXiv 1711 04623"},{"key":"ref38","first-page":"1731","article-title":"Train longer, generalize better: Closing the generalization gap in large batch training of neural networks","author":"hoffer","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref71","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"duchi","year":"2011","journal-title":"J Mach Learn Res"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1088\/1742-5468\/ab3985"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref72","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2014","journal-title":"arXiv 1412 6980"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2804907"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/34.291440"},{"key":"ref23","first-page":"1189","article-title":"Self-paced learning for latent variable models","author":"kumar","year":"2010","journal-title":"Proc Adv Neural Inf Process Syst"},{"journal-title":"UCI Machine Learning Repository","year":"2017","author":"dua","key":"ref67"},{"key":"ref26","article-title":"Prioritized experience replay","author":"schaul","year":"2015","journal-title":"arXiv 1511 05952"},{"key":"ref25","article-title":"Variance reduction in SGD by distributed importance sampling","author":"alain","year":"2015","journal-title":"arXiv 1511 06481"},{"article-title":"Learning multiple layers of features from tiny images","year":"2009","author":"krizhevsky","key":"ref69"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-015-0864-7"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/ITW.2015.7133169"},{"key":"ref63","article-title":"The information bottleneck method","author":"tishby","year":"2000","journal-title":"Arxiv Preprint Physics\/0004057"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553380"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRev.36.823"},{"key":"ref21","first-page":"962","article-title":"Importance sampling for minibatches","volume":"19","author":"csiba","year":"2018","journal-title":"J Mach Learn Res"},{"key":"ref65","article-title":"Opening the black box of deep neural networks via information","author":"shwartz-ziv","year":"2017","journal-title":"arXiv 1703 00810"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1013"},{"key":"ref27","article-title":"Online learning to sample","author":"bouchard","year":"2015","journal-title":"arXiv 1506 09016"},{"key":"ref29","article-title":"Learning what data to learn","author":"fan","year":"2017","journal-title":"arXiv 1702 08635"},{"key":"ref60","article-title":"A deeper look at the Hessian eigenspectrum of deep neural networks and its applications to regularization","author":"sankar","year":"2020","journal-title":"arXiv 2012 03801"},{"key":"ref62","first-page":"2579","article-title":"Visualizing data using t-SNE","volume":"9","author":"van der maaten","year":"2008","journal-title":"J Mach Learn Res"},{"key":"ref61","article-title":"Drill the cork of information bottleneck by inputting the most important data","author":"peng","year":"2021","journal-title":"IEEE Trans Neural Netw Learn Syst"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5962385\/10273172\/09712856.pdf?arnumber=9712856","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,23]],"date-time":"2023-10-23T18:16:44Z","timestamp":1698085004000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9712856\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10]]},"references-count":73,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2022.3147031","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"type":"print","value":"2162-237X"},{"type":"electronic","value":"2162-2388"}],"subject":[],"published":{"date-parts":[[2023,10]]}}}