{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T05:27:19Z","timestamp":1782970039749,"version":"3.54.5"},"reference-count":32,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,6,4]],"date-time":"2023-06-04T00:00:00Z","timestamp":1685836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,6,4]],"date-time":"2023-06-04T00:00:00Z","timestamp":1685836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,6,4]]},"DOI":"10.1109\/icassp49357.2023.10096946","type":"proceedings-article","created":{"date-parts":[[2023,5,5]],"date-time":"2023-05-05T17:28:30Z","timestamp":1683307710000},"page":"1-5","source":"Crossref","is-referenced-by-count":4,"title":["Weighted Sampling for Masked Language Modeling"],"prefix":"10.1109","author":[{"given":"Linhan","family":"Zhang","sequence":"first","affiliation":[{"name":"University of New South Wales,School of Computer and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qian","family":"Chen","sequence":"additional","affiliation":[{"name":"Alibaba Group,Speech Lab of DAMO Academy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wen","family":"Wang","sequence":"additional","affiliation":[{"name":"Alibaba Group,Speech Lab of DAMO Academy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chong","family":"Deng","sequence":"additional","affiliation":[{"name":"Alibaba Group,Speech Lab of DAMO Academy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xin","family":"Cao","sequence":"additional","affiliation":[{"name":"University of New South Wales,School of Computer and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kongzhang","family":"Hao","sequence":"additional","affiliation":[{"name":"University of New South Wales,School of Computer and Engineering"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuxin","family":"Jiang","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology (Guangzhou),China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Wang","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology (Guangzhou),China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.262"},{"key":"ref12","first-page":"23102","article-title":"COCO-LM: correcting and contrasting text sequences for language model pretraining","author":"meng","year":"2021","journal-title":"NeurIPS"},{"key":"ref15","first-page":"32","article-title":"*sem 2013 shared task: Semantic textual similarity","author":"agirre","year":"2013","journal-title":"Proceedings of the Second Joint Conference on Lexical and Computational Semantics *SEM"},{"key":"ref14","first-page":"385","article-title":"Semeval-2012 task 6: A pilot on semantic textual similarity","author":"agirre","year":"2012","journal-title":"NAACL-HLT"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-naacl.191"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.223"},{"key":"ref11","article-title":"ELECTRA: pre-training text encoders as discriminators rather than generators","author":"clark","year":"2020","journal-title":"ICLRE"},{"key":"ref10","article-title":"ALBERT: A lite BERT for selfsupervised learning of language representations","author":"lan","year":"2020","journal-title":"ICLRE"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.393"},{"key":"ref2","article-title":"Roberta: A robustly optimized BERT pretraining approach","author":"liu","year":"2019","journal-title":"CoRR"},{"key":"ref1","first-page":"4171","article-title":"BERT: pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"0","journal-title":"NAACL-HLT"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S15-2045"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/S14-2010"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S17-2001"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S16-1081"},{"key":"ref24","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"radford","year":"2019","journal-title":"OpenAIRE blog"},{"key":"ref23","article-title":"Improving language understanding by generative pre-training","author":"radford","year":"2018"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.20"},{"key":"ref25","article-title":"Language models are few-shot learners","author":"brown","year":"2020","journal-title":"NeurIPS"},{"key":"ref20","first-page":"216","article-title":"A SICK cure for the evaluation of compositional distributional semantic models","author":"marelli","year":"2014","journal-title":"LREC"},{"key":"ref22","article-title":"Whitening sentence representations for better semantics and faster retrieval","author":"su","year":"2021","journal-title":"CoRR"},{"key":"ref21","article-title":"GLUE: A multi-task benchmark and analysis platform for natural language understanding","author":"wang","year":"2019","journal-title":"ICLRE"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.552"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.295"},{"key":"ref29","first-page":"3021","article-title":"Improved universal sentence embeddings with prompt-based contrastive learning and energybased learning","author":"jiang","year":"2022","journal-title":"Findings EMNLP"},{"key":"ref8","first-page":"2337","article-title":"Efficient training of BERT by progressively stacking","volume":"97","author":"gong","year":"2019","journal-title":"ICML"},{"key":"ref7","article-title":"Megatron-lm: Training multibillion parameter language models using model parallelism","author":"shoeybi","year":"2019","journal-title":"CoRR"},{"key":"ref9","article-title":"Taking notes on the fly helps language pre-training","author":"wu","year":"2021","journal-title":"ICLRE"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1006"},{"key":"ref3","article-title":"Representation degeneration problem in training natural language generation models","author":"gao","year":"2019","journal-title":"ICLRE"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.603"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.733"}],"event":{"name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","location":"Rhodes Island, Greece","start":{"date-parts":[[2023,6,4]]},"end":{"date-parts":[[2023,6,10]]}},"container-title":["ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10094559\/10094560\/10096946.pdf?arnumber=10096946","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,20]],"date-time":"2023-11-20T19:04:17Z","timestamp":1700507057000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10096946\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,4]]},"references-count":32,"URL":"https:\/\/doi.org\/10.1109\/icassp49357.2023.10096946","relation":{},"subject":[],"published":{"date-parts":[[2023,6,4]]}}}