{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,26]],"date-time":"2026-01-26T10:52:00Z","timestamp":1769424720691,"version":"3.49.0"},"reference-count":115,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62206194"],"award-info":[{"award-number":["62206194"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004608","name":"Natural Science Foundation of Jiangsu Province","doi-asserted-by":"publisher","award":["BK20220488"],"award-info":[{"award-number":["BK20220488"]}],"id":[{"id":"10.13039\/501100004608","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2024,8]]},"DOI":"10.1109\/tpami.2024.3370716","type":"journal-article","created":{"date-parts":[[2024,2,29]],"date-time":"2024-02-29T18:55:01Z","timestamp":1709232901000},"page":"5763-5778","source":"Crossref","is-referenced-by-count":6,"title":["Randomness Regularization With Simple Consistency Training for Neural Networks"],"prefix":"10.1109","volume":"46","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6286-7529","authenticated-orcid":false,"given":"Juntao","family":"Li","sequence":"first","affiliation":[{"name":"Soochow University, Suzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1550-2877","authenticated-orcid":false,"given":"Xiaobo","family":"Liang","sequence":"additional","affiliation":[{"name":"Soochow University, Suzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3530-590X","authenticated-orcid":false,"given":"Lijun","family":"Wu","sequence":"additional","affiliation":[{"name":"Microsoft Research, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-7050-9811","authenticated-orcid":false,"given":"Yue","family":"Wang","sequence":"additional","affiliation":[{"name":"Soochow University, Suzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3103-1999","authenticated-orcid":false,"given":"Qi","family":"Meng","sequence":"additional","affiliation":[{"name":"Microsoft Research, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9095-0776","authenticated-orcid":false,"given":"Tao","family":"Qin","sequence":"additional","affiliation":[{"name":"Microsoft Research, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3895-5510","authenticated-orcid":false,"given":"Min","family":"Zhang","sequence":"additional","affiliation":[{"name":"Soochow University, Suzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0476-8020","authenticated-orcid":false,"given":"Tie-Yan","family":"Liu","sequence":"additional","affiliation":[{"name":"Microsoft Research, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.5555\/3104322.3104425"},{"key":"ref2","first-page":"1058","article-title":"Regularization of neural networks using DropConnect","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wan"},{"issue":"1","key":"ref3","first-page":"1929","article-title":"Dropout: A simple way to prevent neural networks from overfitting","volume":"15","author":"Srivastava","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2022.3153264"},{"key":"ref5","first-page":"316","article-title":"Randomness in neural network training: Characterizing the impact of tooling","volume-title":"Proc. Mach. Learn. Syst.","author":"Zhuang"},{"key":"ref6","article-title":"Improving neural networks by preventing co-adaptation of feature detectors","author":"Hinton","year":"2012"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-1162"},{"key":"ref8","article-title":"Reducing transformer depth on demand with structured dropout","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Fan"},{"key":"ref9","article-title":"Dropout with expectation-linear regularization","author":"Ma","year":"2016"},{"key":"ref10","article-title":"Fraternal dropout","author":"Zolna","year":"2018"},{"key":"ref11","first-page":"21997","article-title":"DropGNN: Random dropouts increase the expressiveness of graph neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Papp"},{"key":"ref12","first-page":"10 890","article-title":"R-Drop: Regularized dropout for neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Liang"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.302"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref16","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"Liu","year":"2019"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref18","article-title":"MUSE: Parallel multi-scale attention for sequence to sequence learning","author":"Zhao","year":"2019"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1558"},{"key":"ref20","article-title":"Very deep transformers for neural machine translation","author":"Liu","year":"2020"},{"key":"ref21","first-page":"10 018","article-title":"Data diversification: A simple strategy for neural machine translation","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Nguyen"},{"key":"ref22","article-title":"Incorporating BERT into neural machine translation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhu"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-4009"},{"key":"ref24","article-title":"Teaching machines to read and comprehend","author":"Hermann","year":"2015"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.217"},{"key":"ref27","first-page":"11 328","article-title":"PEGASUS: Pre-training with extracted gap-sentences for abstractive summarization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref28","article-title":"Better fine-tuning by reducing representational collapse","author":"Aghajanyan","year":"2020"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.3115\/1118162.1118168"},{"key":"ref30","first-page":"5753","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Yang"},{"key":"ref31","article-title":"ELECTRA: Pre-training text encoders as discriminators rather than generators","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Clark"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-5446"},{"key":"ref33","article-title":"Pointer sentinel mixture models","author":"Merity","year":"2016"},{"key":"ref34","article-title":"Adaptive input representations for neural language modeling","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Baevski"},{"key":"ref35","article-title":"Enhancing the locality and breaking the memory bottleneck of transformer on time series forecasting","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Li"},{"key":"ref36","article-title":"Reformer: The efficient transformer","author":"Kitaev","year":"2020"},{"key":"ref37","article-title":"Neural machine translation by jointly learning to align and translate","author":"Bahdanau","year":"2014"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210006"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i12.17325"},{"key":"ref40","article-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref42","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dosovitskiy"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58558-7_29"},{"key":"ref44","article-title":"How powerful are graph neural networks?","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Xu"},{"key":"ref45","first-page":"22 118","article-title":"Open graph benchmark: Datasets for machine learning on graphs","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Hu"},{"key":"ref46","first-page":"1310","article-title":"Certified adversarial robustness via randomized smoothing","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cohen"},{"key":"ref47","article-title":"Self-distillation amplifies regularization in Hilbert space","author":"Mobahi","year":"2020"},{"key":"ref48","article-title":"Graph warp module: An auxiliary module for boosting the power of graph neural networks","author":"Ishiguro","year":"2019"},{"key":"ref49","article-title":"Strategies for pre-training graph neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hu"},{"key":"ref50","article-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2014"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045167"},{"key":"ref52","first-page":"2790","article-title":"Parameter-efficient transfer learning for NLP","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Houlsby"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.39"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acllong.353"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.243"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2023.08.012"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-short.1"},{"key":"ref58","article-title":"LoRA: Low-rank adaptation of large language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hu"},{"key":"ref59","first-page":"1631","article-title":"Recursive deep models for semantic compositionality over a sentiment treebank","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Socher"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/11736790_9"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"ref62","first-page":"649","article-title":"Character-level convolutional networks for text classification","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhang"},{"key":"ref63","first-page":"142","article-title":"Learning word vectors for sentiment analysis","volume-title":"Proc. 49th Annu. Meeting Assoc. Comput. Linguistics: Hum. Lang. Technol.","author":"Maas"},{"key":"ref64","article-title":"Improving language understanding by generative pre-training","author":"Radford","year":"2018","journal-title":"OpenAI Blog"},{"issue":"8","key":"ref65","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref66","article-title":"Language models are few-shot learners","author":"Brown","year":"2020"},{"key":"ref67","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","author":"Fedus","year":"2021"},{"key":"ref68","article-title":"Survey of dropout methods for deep neural networks","author":"Labach","year":"2019"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103589"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00769"},{"key":"ref71","first-page":"529","article-title":"Simplifying neural nets by discovering flat minima","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Hochreiter"},{"key":"ref72","article-title":"Analyzing noise in autoencoders and deep networks","author":"Poole","year":"2014"},{"key":"ref73","first-page":"3084","article-title":"Adaptive dropout for training deep neural networks","volume-title":"Proc. 26th Int. Conf. Neural Inf. Process. Syst.","author":"Ba"},{"key":"ref74","first-page":"118","article-title":"Fast dropout training","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wang"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.5555\/3045390.3045502"},{"key":"ref76","first-page":"2498","article-title":"Variational dropout sparsifies deep neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Molchanov"},{"key":"ref77","first-page":"6778","article-title":"Structured Bayesian pruning via log-normal multiplicative noise","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Neklyudov"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-019-09784-7"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i11.17127"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2015.07.007"},{"key":"ref81","article-title":"Improved regularization of convolutional neural networks with cutout","author":"DeVries","year":"2017"},{"key":"ref82","first-page":"1019","article-title":"A theoretically grounded application of dropout in recurrent neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Gal"},{"key":"ref83","first-page":"1757","article-title":"Recurrent dropout without memory loss","volume-title":"Proc. 26th Int. Conf. Comput. Linguistics: Tech. Papers","author":"Semeniuta"},{"key":"ref84","article-title":"Regularizing and optimizing LSTM language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Merity"},{"key":"ref85","article-title":"DropAttention: A regularization method for fully-connected self-attention networks","author":"Zehui","year":"2019"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.178"},{"key":"ref87","article-title":"Not all attention is all you need","author":"Wu","year":"2021"},{"key":"ref88","article-title":"Adversarial dropout regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Saito"},{"key":"ref89","first-page":"950","article-title":"A simple weight decay can improve generalization","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Krogh"},{"key":"ref90","first-page":"1097","article-title":"ImageNet classification with deep convolutional neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Krizhevsky"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v30i1.10202"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.5555\/3157096.3157329"},{"key":"ref93","first-page":"901","article-title":"Weight normalization: A simple reparameterization to accelerate training of deep neural networks","volume-title":"Proc. 30th Int. Conf. Neural Inf. Process. Syst.","author":"Salimans"},{"key":"ref94","article-title":"Layer normalization","author":"Ba","year":"2016"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11768"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01261-8_1"},{"key":"ref97","first-page":"153","article-title":"The difficulty of training deep architectures and the effect of unsupervised pre-training","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Erhan"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.123"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref100","article-title":"A simple but tough-to-beat data augmentation approach for natural language understanding and generation","author":"Shen","year":"2020"},{"key":"ref101","article-title":"Distilling the knowledge in a neural network","author":"Hinton","year":"2015"},{"key":"ref102","first-page":"1607","article-title":"Born again neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Furlanello"},{"key":"ref103","article-title":"Towards understanding ensemble, knowledge distillation and self-distillation in deep learning","author":"Allen-Zhu","year":"2020"},{"key":"ref104","article-title":"MIXKD: Towards efficient distillation of large-scale language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liang"},{"key":"ref105","article-title":"SEED: Self-supervised distillation for visual representation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Fang"},{"key":"ref106","article-title":"Rethinking soft labels for knowledge distillation: A bias-variance tradeoff perspective","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhou"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20083-0_27"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.26615\/978-954-452-056-4_050"},{"key":"ref109","article-title":"A closer look at deep learning heuristics: Learning rate restarts, warmup and distillation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Gotmare"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00381"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00454"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1162"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-6301"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-6319"},{"key":"ref115","first-page":"10 181","article-title":"The implicit and explicit regularization effects of dropout","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wei"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10582780\/10453595.pdf?arnumber=10453595","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T17:31:59Z","timestamp":1720027919000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10453595\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8]]},"references-count":115,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3370716","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,8]]}}}