{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T16:21:24Z","timestamp":1781713284811,"version":"3.54.5"},"reference-count":100,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,5,1]],"date-time":"2022-05-01T00:00:00Z","timestamp":1651363200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100000266","name":"Engineering and Physical Sciences Research Council","doi-asserted-by":"publisher","award":["EP\/S000631\/1"],"award-info":[{"award-number":["EP\/S000631\/1"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000266","name":"Engineering and Physical Sciences Research Council","doi-asserted-by":"publisher","award":["EP\/L016427\/1"],"award-info":[{"award-number":["EP\/L016427\/1"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000266","name":"Engineering and Physical Sciences Research Council","doi-asserted-by":"publisher","award":["EP\/R026173\/1"],"award-info":[{"award-number":["EP\/R026173\/1"]}],"id":[{"id":"10.13039\/501100000266","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Signal Process. Mag."],"published-print":{"date-parts":[[2022,5]]},"DOI":"10.1109\/msp.2021.3134634","type":"journal-article","created":{"date-parts":[[2022,5,6]],"date-time":"2022-05-06T19:52:24Z","timestamp":1651866744000},"page":"42-62","source":"Crossref","is-referenced-by-count":373,"title":["Self-Supervised Representation Learning: Introduction, advances, and challenges"],"prefix":"10.1109","volume":"39","author":[{"given":"Linus","family":"Ericsson","sequence":"first","affiliation":[{"name":"MScR degree, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Henry","family":"Gouk","sequence":"additional","affiliation":[{"name":"School of Informatics, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chen Change","family":"Loy","sequence":"additional","affiliation":[{"name":"Associate Professor, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Timothy M.","family":"Hospedales","sequence":"additional","affiliation":[{"name":"Artificial Intelligence, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","article-title":"Barlow twins: Self-supervised learning via redundancy reduction","author":"zbontar","year":"2021"},{"key":"ref38","first-page":"297","article-title":"Noise-contrastive estimation: A new estimation principle for unnormalized statistical models","author":"gutmann","year":"0","journal-title":"Mach Learn Res"},{"key":"ref33","article-title":"Unsupervised representation learning by predicting image rotations","author":"gidaris","year":"2018"},{"key":"ref32","article-title":"Language models are few-shot learners","author":"brown","year":"2020"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.372"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00202"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2006.100"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_12"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053985"},{"key":"ref28","article-title":"Using self-supervised learning can improve model robustness and uncertainty","author":"hendrycks","year":"2019"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403237"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00537"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-019-05855-6"},{"key":"ref22","first-page":"3111","article-title":"Distributed representations of words and phrases and their compositionality","author":"mikolov","year":"0","journal-title":"Proc 26th Int Conf Neural Inf Process Syst NeurIPS"},{"key":"ref21","author":"hastie","year":"2009","journal-title":"The Elements of Statistical Learning Data Mining Inference and Prediction"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00719"},{"key":"ref23","article-title":"Discriminative unsupervised feature learning with exemplar convolutional neural networks","author":"dosovitskiy","year":"2014"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.278"},{"key":"ref25","article-title":"Cross-lingual language model pretraining","author":"conneau","year":"2019"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.426"},{"key":"ref50","article-title":"Predicting what you already know helps: Provable self-supervised learning","author":"lee","year":"2020"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1561\/2200000008"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/2812802"},{"key":"ref58","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","author":"dosovitskiy","year":"0","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref57","first-page":"1025","article-title":"Inductive representation learning on large graphs","author":"hamilton","year":"0","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1202"},{"key":"ref55","first-page":"13","article-title":"Vilbert: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","author":"lu","year":"0","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6840"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46493-0_35"},{"key":"ref52","first-page":"1691","article-title":"Generative pretraining from pixels","author":"chen","year":"0","journal-title":"Proc 37th Int Conf Mach Learn"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00975"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2992393"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.97"},{"key":"ref6","article-title":"Bootstrap your own latent: A new approach to self-supervised learning","author":"grill","year":"2020"},{"key":"ref5","article-title":"Big Self-Supervised models are strong semi-supervised learners","author":"chen","year":"2020"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01058"},{"key":"ref7","article-title":"Unsupervised learning of visual features by contrasting cluster assignments","author":"caron","year":"2020"},{"key":"ref49","article-title":"A mathematical exploration of why language models help solve downstream tasks","author":"saunshi","year":"2021"},{"key":"ref9","article-title":"Self-supervised learning by cross-modal audio-video clustering","author":"alwassel","year":"2020"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"ref45","article-title":"Debiased contrastive learning","author":"chuang","year":"2020"},{"key":"ref48","first-page":"5628","article-title":"A theoretical analysis of contrastive unsupervised representation learning","author":"saunshi","year":"0","journal-title":"Proc 36th Int Conf Mach Learn"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00672"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01231-1_39"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58621-8_45"},{"key":"ref44","article-title":"What makes for good views for contrastive learning?","author":"tian","year":"2020"},{"key":"ref43","article-title":"Learning transferable visual models from natural language supervision","author":"radford","year":"2021"},{"key":"ref73","first-page":"12,884","article-title":"Benchmarking representation learning on natural world image collections","author":"mac aodha","year":"0","journal-title":"Proc IEEE Conf Comput Vis and Pattern Recog"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/bti1007"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1018"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330785"},{"key":"ref76","article-title":"3d self-supervised methods for medical imaging","author":"taleb","year":"2020"},{"key":"ref77","article-title":"Deep anomaly detection using geometric transformations","author":"golan","year":"2018"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/LGRS.2020.3038420"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2019.101539"},{"key":"ref78","article-title":"Self-supervised co-training for video representation learning","author":"han","year":"2020"},{"key":"ref79","first-page":"7","article-title":"Self-supervised multimodal versatile networks","author":"alayrac","year":"0","journal-title":"Proc Conf Neural Inf Process Syst"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00914"},{"key":"ref62","year":"0","journal-title":"Youtube-8m A large and diverse labeled video dataset for video understanding research"},{"key":"ref61","article-title":"The kinetics human action video dataset","author":"kay","year":"2017"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00272"},{"key":"ref64","article-title":"Pointer sentinel mixture models","author":"merity","year":"2017"},{"key":"ref65","author":"gokaslan","year":"2019","journal-title":"OpenWebText Corpus"},{"key":"ref66","first-page":"4","article-title":"N-gram counts and language models from the common crawl","author":"buck","year":"0","journal-title":"Proc Int Conf Lang Resources Eval"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9052942"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7952261"},{"key":"ref2","first-page":"6000","article-title":"Attention is all you need","author":"vaswani","year":"0","journal-title":"Proc 31st Int Conf Neural Inf Process Syst NIPS’17"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref95","article-title":"Demystifying contrastive self-supervised learning: invariances, augmentations and dataset biases","author":"purushwalkam","year":"2020"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1145\/3381831"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btx252"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1039\/C7SC02664A"},{"key":"ref91","article-title":"Strategies for pre-training graph neural networks","author":"hu","year":"2020"},{"key":"ref90","article-title":"InfoGraph: Unsupervised and semi-supervised graph-level representation learning via mutual information maximization","author":"sun","year":"2020"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054548"},{"key":"ref99","article-title":"Rethinking pre-training and self-training","author":"zoph","year":"2020"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00267"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2858"},{"key":"ref10","article-title":"Wav2vec 2.0: A framework for self-supervised learning of speech representations","author":"baevski","year":"2020"},{"key":"ref11","article-title":"Representation learning with contrastive predictive coding","author":"van den oord","year":"2018"},{"key":"ref12","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2019"},{"key":"ref13","author":"radford","year":"2019","journal-title":"Language Models are Unsupervised Multitask Learners"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939754"},{"key":"ref15","article-title":"Deep graph infomax","author":"velic?kovic\u00b4","year":"2019"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00649"},{"key":"ref82","article-title":"Reducing sentiment bias in language models via counterfactual evaluation","author":"huang","year":"2019"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01255"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"ref18","author":"goodfellow","year":"2016","journal-title":"Deep Learning"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1473"},{"key":"ref19","article-title":"Distance-based regularisation of deep networks for fine-tuning","author":"gouk","year":"2021"},{"key":"ref83","first-page":"173","article-title":"Deep speech 2: End-to-end speech recognition in English and Mandarin","author":"amodei","year":"0","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref80","article-title":"SuperGLUE: A stickier benchmark for general-purpose language understanding systems","author":"wang","year":"2019"},{"key":"ref89","article-title":"Semi-supervised classification with graph convolutional networks","author":"kipf","year":"2017"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2605"},{"key":"ref86","article-title":"Unsupervised scalable representation learning for multivariate time series","author":"franceschi","year":"2019"},{"key":"ref87","article-title":"Subject-aware contrastive learning for biosignals","author":"cheng","year":"2020"},{"key":"ref88","article-title":"Deep transformer models for time series forecasting: The influenza prevalence case","author":"wu","year":"2020"}],"container-title":["IEEE Signal Processing Magazine"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/79\/9770265\/09770283.pdf?arnumber=9770283","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,6]],"date-time":"2022-06-06T20:16:26Z","timestamp":1654546586000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9770283\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5]]},"references-count":100,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/msp.2021.3134634","relation":{},"ISSN":["1053-5888","1558-0792"],"issn-type":[{"value":"1053-5888","type":"print"},{"value":"1558-0792","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5]]}}}