{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T21:16:36Z","timestamp":1757625396133,"version":"3.44.0"},"publisher-location":"Cham","reference-count":27,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783032025500"},{"type":"electronic","value":"9783032025517"}],"license":[{"start":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T00:00:00Z","timestamp":1755820800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T00:00:00Z","timestamp":1755820800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-02551-7_19","type":"book-chapter","created":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T05:27:40Z","timestamp":1755754060000},"page":"216-227","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Enhancing Masked Language Modeling in\u00a0BERT Models Using Pretrained Static Embeddings"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-1019-9218","authenticated-orcid":false,"given":"Adam","family":"Mi\u0161tera","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3096-675X","authenticated-orcid":false,"given":"Pavel","family":"Kr\u00e1l","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,8,22]]},"reference":[{"key":"19_CR1","doi-asserted-by":"crossref","unstructured":"Alghanmi, I., Espinosa\u00a0Anke, L., Schockaert, S.: Combining BERT with static word embeddings for categorizing social media. In: Xu, W., Ritter, A., Baldwin, T., Rahimi, A. (eds.) Proceedings of the Sixth Workshop on Noisy User-generated Text (W-NUT 2020), pp. 28\u201333. Association for Computational Linguistics, Online (2020)","DOI":"10.18653\/v1\/2020.wnut-1.5"},{"key":"19_CR2","doi-asserted-by":"crossref","unstructured":"Artetxe, M., Labaka, G., Agirre, E.: Learning principled bilingual mappings of word embeddings while preserving monolingual invariance. In: Su, J., Duh, K., Carreras, X. (eds.) Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, pp. 2289\u20132294. Association for Computational Linguistics, Austin (2016)","DOI":"10.18653\/v1\/D16-1250"},{"key":"19_CR3","doi-asserted-by":"crossref","unstructured":"Bhargava, P., Drozd, A., Rogers, A.: Generalization in NLI: ways (not) to go beyond simple heuristics (2021)","DOI":"10.18653\/v1\/2021.insights-1.18"},{"key":"19_CR4","doi-asserted-by":"publisher","first-page":"135","DOI":"10.1162\/tacl_a_00051","volume":"5","author":"P Bojanowski","year":"2017","unstructured":"Bojanowski, P., Grave, E., Joulin, A., Mikolov, T.: Enriching word vectors with subword information. Trans. Assoc. Comput. Linguist. 5, 135\u2013146 (2017)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"19_CR5","unstructured":"Clark, K., Luong, M., Le, Q.V., Manning, C.D.: ELECTRA: pre-training text encoders as discriminators rather than generators. CoRR abs\/2003.10555 (2020)"},{"key":"19_CR6","doi-asserted-by":"crossref","unstructured":"Cliche, M.: BB_twtr at SemEval-2017 task 4: twitter sentiment analysis with CNNs and LSTMs. In: Bethard, S., Carpuat, M., Apidianaki, M., Mohammad, S.M., Cer, D., Jurgens, D. (eds.) Proceedings of the 11th International Workshop on Semantic Evaluation (SemEval-2017), pp. 573\u2013580. Association for Computational Linguistics, Vancouver (2017)","DOI":"10.18653\/v1\/S17-2094"},{"key":"19_CR7","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Burstein, J., Doran, C., Solorio, T. (eds.) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186. Association for Computational Linguistics, Minneapolis (2019)"},{"issue":"2","key":"19_CR8","first-page":"31","volume":"100","author":"EM Dharma","year":"2022","unstructured":"Dharma, E.M., Gaol, F.L., Warnars, H., Soewito, B.: The accuracy comparison among word2vec, glove, and fasttext towards convolution neural network (CNN) text classification. J. Theor. Appl. Inf. Technol. 100(2), 31 (2022)","journal-title":"J. Theor. Appl. Inf. Technol."},{"key":"19_CR9","doi-asserted-by":"crossref","unstructured":"D\u2019Sa, A.G., Illina, I., Fohr, D.: BERT and fastText embeddings for automatic detection of toxic speech. In: 2020 International Multi-Conference on: \u201cOrganization of Knowledge and Advanced Technologies\u201d (OCTA), pp.\u00a01\u20135 (2020)","DOI":"10.1109\/OCTA49274.2020.9151853"},{"key":"19_CR10","unstructured":"Grave, E., Bojanowski, P., Gupta, P., Joulin, A., Mikolov, T.: Learning word vectors for 157 languages. In: Proceedings of the International Conference on Language Resources and Evaluation (LREC 2018) (2018)"},{"issue":"23","key":"19_CR11","doi-asserted-by":"publisher","first-page":"146","DOI":"10.1080\/00437956.1954.11659520","volume":"10","author":"Z Harris","year":"1954","unstructured":"Harris, Z.: Distributional structure. Word 10(23), 146\u2013162 (1954)","journal-title":"Word"},{"issue":"2","key":"19_CR12","doi-asserted-by":"publisher","first-page":"195","DOI":"10.34288\/jri.v5i2.507","volume":"5","author":"MR Ilham","year":"2023","unstructured":"Ilham, M.R., Laksito, A.D.: Comparative analysis of using word embedding in deep learning for text classification. J. Riset Inform. 5(2), 195\u2013202 (2023)","journal-title":"J. Riset Inform."},{"key":"19_CR13","doi-asserted-by":"crossref","unstructured":"Jiao, X., et al.: TinyBERT: distilling BERT for natural language understanding. arXiv preprint arXiv:1909.10351 (2019)","DOI":"10.18653\/v1\/2020.findings-emnlp.372"},{"key":"19_CR14","doi-asserted-by":"crossref","unstructured":"Jo, J.y., Myaeng, S.H.: Roles and utilization of attention heads in transformer-based neural language models. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 3404\u20133417 (2020)","DOI":"10.18653\/v1\/2020.acl-main.311"},{"key":"19_CR15","doi-asserted-by":"crossref","unstructured":"Lample, G., Ballesteros, M., Subramanian, S., Kawakami, K., Dyer, C.: Neural architectures for named entity recognition. In: Knight, K., Nenkova, A., Rambow, O. (eds.) Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 260\u2013270. Association for Computational Linguistics, San Diego (2016)","DOI":"10.18653\/v1\/N16-1030"},{"key":"19_CR16","unstructured":"Liu, Y., et al.: RoBERTa: a robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692 (2019)"},{"key":"19_CR17","unstructured":"Loshchilov, I., Hutter, F.: Decoupled weight decay regularization. arXiv preprint arXiv:1711.05101 (2017)"},{"key":"19_CR18","unstructured":"Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space. CoRR abs\/1301.3781 (2013)"},{"key":"19_CR19","doi-asserted-by":"crossref","unstructured":"Pennington, J., Socher, R., Manning, C.: GloVe: global vectors for word representation. In: Moschitti, A., Pang, B., Daelemans, W. (eds.) Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 1532\u20131543. Association for Computational Linguistics, Doha (2014)","DOI":"10.3115\/v1\/D14-1162"},{"key":"19_CR20","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. CoRR abs\/1910.10683 (2019)"},{"key":"19_CR21","unstructured":"Sanh, V., Debut, L., Chaumond, J., Wolf, T.: DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108 (2019)"},{"key":"19_CR22","doi-asserted-by":"crossref","unstructured":"Sun, Z., Yu, H., Song, X., Liu, R., Yang, Y., Zhou, D.: MobileBERT: a compact task-agnostic BERT for resource-limited devices. arXiv preprint arXiv:2004.02984 (2020)","DOI":"10.18653\/v1\/2020.acl-main.195"},{"key":"19_CR23","unstructured":"Turc, I., Chang, M., Lee, K., Toutanova, K.: Well-read students learn better: the impact of student initialization on knowledge distillation. CoRR abs\/1908.08962 (2019)"},{"key":"19_CR24","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Guyon, I., Luxburg, U.V., Bengio, S., Wallach, H., Fergus, R., Vishwanathan, S., Garnett, R. (eds.) Advances in Neural Information Processing Systems, vol.\u00a030. Curran Associates, Inc. (2017)"},{"key":"19_CR25","doi-asserted-by":"crossref","unstructured":"Warner, B., et al.: Smarter, better, faster, longer: a modern bidirectional encoder for fast, memory efficient, and long context finetuning and inference (2024)","DOI":"10.18653\/v1\/2025.acl-long.127"},{"key":"19_CR26","doi-asserted-by":"crossref","unstructured":"Wu, C., Wu, F., Chen, Y., Wu, S., Yuan, Z., Huang, Y.: Neural metaphor detecting with CNN-LSTM model. In: Beigman\u00a0Klebanov, B., Shutova, E., Lichtenstein, P., Muresan, S., Wee, C. (eds.) Proceedings of the Workshop on Figurative Language Processing, pp. 110\u2013114. Association for Computational Linguistics, New Orleans (2018)","DOI":"10.18653\/v1\/W18-0913"},{"key":"19_CR27","doi-asserted-by":"crossref","unstructured":"Zhuang, B., Liu, J., Pan, Z., He, H., Weng, Y., Shen, C.: A survey on efficient training of transformers. arXiv preprint arXiv:2302.01107 (2023)","DOI":"10.24963\/ijcai.2023\/764"}],"container-title":["Lecture Notes in Computer Science","Text, Speech, and Dialogue"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-02551-7_19","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,9]],"date-time":"2025-09-09T18:04:48Z","timestamp":1757441088000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-02551-7_19"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,22]]},"ISBN":["9783032025500","9783032025517"],"references-count":27,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-02551-7_19","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2025,8,22]]},"assertion":[{"value":"22 August 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"TSD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Text, Speech, and Dialogue","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Erlangen","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25 August 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 August 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"tsd2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.kiv.zcu.cz\/tsd2025\/index.php","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}