{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T15:18:36Z","timestamp":1782832716822,"version":"3.54.5"},"publisher-location":"Cham","reference-count":79,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783031243486","type":"print"},{"value":"9783031243493","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-24349-3_6","type":"book-chapter","created":{"date-parts":[[2023,4,3]],"date-time":"2023-04-03T10:09:17Z","timestamp":1680516557000},"page":"81-105","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Transformers in\u00a0Natural Language Processing"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7972-7442","authenticated-orcid":false,"given":"Fran\u00e7ois","family":"Yvon","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,4,4]]},"reference":[{"key":"6_CR1","doi-asserted-by":"publisher","unstructured":"Aharoni, R., Johnson, M., Firat, O.: Massively multilingual neural machine translation. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 3874\u20133884. Association for Computational Linguistics, Minneapolis (2019). https:\/\/doi.org\/10.18653\/v1\/N19-1388","DOI":"10.18653\/v1\/N19-1388"},{"key":"6_CR2","unstructured":"Baevski, A., Zhou, Y., Mohamed, A., Auli, M.: Wav2vec 2.0: a framework for self-supervised learning of speech representations. In: Larochelle, H., Ranzato, M., Hadsell, R., Balcan, M.F., Lin, H. (eds.) Advances in Neural Information Processing Systems, vol. 33, pp. 12449\u201312460. Curran Associates, Inc. (2020)"},{"key":"6_CR3","unstructured":"Bahdanau, D., Cho, K., Bengio, Y.: Neural machine translation by jointly learning to align and translate. In: Proceedings of the first International Conference on Learning Representations. ICLR, San Diego (2015)"},{"key":"6_CR4","doi-asserted-by":"publisher","unstructured":"Barbieri, F., Camacho-Collados, J., Espinosa Anke, L., Neves, L.: TweetEval: unified benchmark and comparative evaluation for tweet classification. In: Findings of the Association for Computational Linguistics: EMNLP 2020, pp. 1644\u20131650. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.findings-emnlp.148","DOI":"10.18653\/v1\/2020.findings-emnlp.148"},{"key":"6_CR5","doi-asserted-by":"publisher","unstructured":"Beltagy, I., Lo, K., Cohan, A.: SciBERT: a pretrained language model for scientific text. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 3615\u20133620. Association for Computational Linguistics, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1371","DOI":"10.18653\/v1\/D19-1371"},{"key":"6_CR6","doi-asserted-by":"publisher","unstructured":"Bender, E.M., Gebru, T., McMillan-Major, A., Shmitchell, S.: On the dangers of stochastic parrots: can language models be too big? In: Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency, FAccT 2021, pp. 610\u2013623. Association for Computing Machinery, New York (2021). https:\/\/doi.org\/10.1145\/3442188.3445922","DOI":"10.1145\/3442188.3445922"},{"key":"6_CR7","doi-asserted-by":"publisher","unstructured":"Bender, E.M., Koller, A.: Climbing towards NLU: on meaning, form, and understanding in the age of data. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 5185\u20135198. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.463","DOI":"10.18653\/v1\/2020.acl-main.463"},{"key":"6_CR8","unstructured":"Bengio, S., Vinyals, O., Jaitly, N., Shazeer, N.: Scheduled sampling for sequence prediction with recurrent neural networks. In: Cortes, C., Lawrence, N., Lee, D., Sugiyama, M., Garnett, R. (eds.) Advances in Neural Information Processing Systems, vol. 28. Curran Associates, Inc. (2015)"},{"key":"6_CR9","first-page":"1137","volume":"3","author":"Y Bengio","year":"2003","unstructured":"Bengio, Y., Ducharme, R., Vincent, P., Janvin, C.: A neural probabilistic language model. J. Mach. Learn. Res. 3, 1137\u20131155 (2003)","journal-title":"J. Mach. Learn. Res."},{"key":"6_CR10","unstructured":"B\u00e9rard, A., Pietquin, O., Besacier, L., Servan, C.: Listen and translate: a proof of concept for end-to-end speech-to-text translation. In: NIPS Workshop on End-to-End Learning for Speech and Audio Processing, Barcelona, Spain (2016)"},{"key":"6_CR11","doi-asserted-by":"crossref","unstructured":"Bojanowski, P., Grave, E., Joulin, A., Mikolov, T.: Enriching word vectors with subword information. arXiv preprint arXiv:1607.04606 (2016)","DOI":"10.1162\/tacl_a_00051"},{"key":"6_CR12","unstructured":"Bommasani, R., et al.: On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258 (2021)"},{"key":"6_CR13","doi-asserted-by":"publisher","unstructured":"Bosselut, A., Rashkin, H., Sap, M., Malaviya, C., Celikyilmaz, A., Choi, Y.: COMET: commonsense transformers for automatic knowledge graph construction. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 4762\u20134779. Association for Computational Linguistics, Florence (2019). https:\/\/doi.org\/10.18653\/v1\/P19-1470","DOI":"10.18653\/v1\/P19-1470"},{"key":"6_CR14","unstructured":"Brown, T.B., et al.: Language models are few-shot learners (2020)"},{"key":"6_CR15","unstructured":"Ca\u00f1ete, J., Chaperon, G., Fuentes, R., Ho, J.H., Kang, H., P\u00e9rez, J.: Spanish pre-trained BERT model and evaluation data. In: PML4DC at ICLR 2020 (2020)"},{"key":"6_CR16","doi-asserted-by":"publisher","unstructured":"Chan, B., Schweter, S., M\u00f6ller, T.: German\u2019s next language model. In: Proceedings of the 28th International Conference on Computational Linguistics, pp. 6788\u20136796. International Committee on Computational Linguistics, Barcelona (2020). https:\/\/doi.org\/10.18653\/v1\/2020.coling-main.598","DOI":"10.18653\/v1\/2020.coling-main.598"},{"key":"6_CR17","volume-title":"Statistical Language Learning","author":"E Charniak","year":"1993","unstructured":"Charniak, E.: Statistical Language Learning. The MIT Press, Cambridge (1993)"},{"key":"6_CR18","doi-asserted-by":"crossref","unstructured":"Cho, K., et al.: Learning phrase representations using RNN encoder-decoder for statistical machine translation. In: Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 1724\u20131734. Association for Computational Linguistics, Doha (2014)","DOI":"10.3115\/v1\/D14-1179"},{"key":"6_CR19","doi-asserted-by":"publisher","unstructured":"Cho, K., van Merri\u00ebnboer, B., Bahdanau, D., Bengio, Y.: On the properties of neural machine translation: encoder-decoder approaches. In: Proceedings of SSST-8, Eighth Workshop on Syntax, Semantics and Structure in Statistical Translation, pp. 103\u2013111. Association for Computational Linguistics, Doha (2014). https:\/\/doi.org\/10.3115\/v1\/W14-4012","DOI":"10.3115\/v1\/W14-4012"},{"issue":"Aug","key":"6_CR20","first-page":"2493","volume":"12","author":"R Collobert","year":"2011","unstructured":"Collobert, R., Weston, J., Bottou, L., Karlen, M., Kavukcuoglu, K., Kuksa, P.: Natural language processing (almost) from scratch. J. Mach. Learn. Res. 12(Aug), 2493\u20132537 (2011)","journal-title":"J. Mach. Learn. Res."},{"key":"6_CR21","unstructured":"Conneau, A., Lample, G.: Cross-lingual language model pretraining. In: Wallach, H., Larochelle, H., Beygelzimer, A., d\u2019Alch\u00e9-Buc, F., Fox, E., Garnett, R. (eds.) Advances in Neural Information Processing Systems, vol. 32. Curran Associates, Inc. (2019)"},{"key":"6_CR22","doi-asserted-by":"crossref","unstructured":"Deligne, S., Bimbot, F.: Language modeling by variable length sequences: theoretical formulation and evaluation of multigrams. In: 1995 International Conference onAcoustics, Speech, and Signal Processing, ICASSP-95, vol. 1, pp. 169\u2013172. IEEE (1995)","DOI":"10.1109\/ICASSP.1995.479391"},{"key":"6_CR23","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186. Association for Computational Linguistics, Minneapolis (2019)"},{"key":"6_CR24","doi-asserted-by":"crossref","unstructured":"Dong, L., Xu, S., Xu, B.: Speech-transformer: a no-recurrence sequence-to-sequence model for speech recognition. In: 2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 5884\u20135888. IEEE (2018)","DOI":"10.1109\/ICASSP.2018.8462506"},{"key":"6_CR25","unstructured":"Dosovitskiy, A., et al.: An image is worth 16 $$\\times $$ 16 words: transformers for image recognition at scale. In: International Conference on Learning Representations (2021)"},{"key":"6_CR26","volume-title":"Natural Language Processing","author":"J Eisenstein","year":"2019","unstructured":"Eisenstein, J.: Natural Language Processing. The MIT Press, Cambridge (2019)"},{"issue":"2","key":"6_CR27","doi-asserted-by":"publisher","first-page":"179","DOI":"10.1207\/s15516709cog1402_1","volume":"14","author":"JL Elman","year":"1990","unstructured":"Elman, J.L.: Finding structure in time. Cogn. Sci. 14(2), 179\u2013211 (1990). https:\/\/doi.org\/10.1207\/s15516709cog1402_1","journal-title":"Cogn. Sci."},{"issue":"107","key":"6_CR28","first-page":"1","volume":"22","author":"A Fan","year":"2021","unstructured":"Fan, A., et al.: Beyond English-centric multilingual machine translation. J. Mach. Learn. Res. 22(107), 1\u201348 (2021)","journal-title":"J. Mach. Learn. Res."},{"key":"6_CR29","unstructured":"Fedus, W., Zoph, B., Shazeer, N.: Switch transformers: scaling to trillion parameter models with simple and efficient sparsity (2021)"},{"key":"6_CR30","doi-asserted-by":"publisher","unstructured":"Firat, O., Cho, K., Bengio, Y.: Multi-way, multilingual neural machine translation with a shared attention mechanism. In: Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 866\u2013875. Association for Computational Linguistics (2016). https:\/\/doi.org\/10.18653\/v1\/N16-1101","DOI":"10.18653\/v1\/N16-1101"},{"issue":"2","key":"6_CR31","first-page":"23","volume":"12","author":"P Gage","year":"1994","unstructured":"Gage, P.: A new algorithm for data compression. Comput. Users J. 12(2), 23\u201338 (1994)","journal-title":"Comput. Users J."},{"key":"6_CR32","unstructured":"Gu, J., Bradbury, J., Xiong, C., Li, V.O.K., Socher, R.: Non-autoregressive neural machine translation. In: Proceedings of the International Conference on Representation Learning, ICLR 2018 (2018)"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"Gulati, A., et al.: Conformer: convolution-augmented transformer for speech recognition. In: Proceedings of the Interspeech 2020, pp. 5036\u20135040 (2020)","DOI":"10.21437\/Interspeech.2020-3015"},{"key":"6_CR34","unstructured":"Ha, T.H., Niehues, J., Waibel, A.: Toward multilingual neural machine translation with universal encoder and decoder. In: Proceedings of the 13th International Workshop on Spoken Language Translation, IWSLT 2016, Vancouver, Canada (2016)"},{"issue":"248","key":"6_CR35","first-page":"1","volume":"21","author":"P Henderson","year":"2020","unstructured":"Henderson, P., Hu, J., Romoff, J., Brunskill, E., Jurafsky, D., Pineau, J.: Towards the systematic reporting of the energy and carbon footprints of machine learning. J. Mach. Learn. Res. 21(248), 1\u201343 (2020)","journal-title":"J. Mach. Learn. Res."},{"issue":"8","key":"6_CR36","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997). https:\/\/doi.org\/10.1162\/neco.1997.9.8.1735","journal-title":"Neural Comput."},{"key":"6_CR37","unstructured":"Jain, A., Ganesamoorty, M.: NukeBERT: a pre-trained language model for low resource nuclear domain. CoRR abs\/2003.13821 (2020)"},{"key":"6_CR38","volume-title":"Statistical Methods for Speech Recognition","author":"F Jelinek","year":"1997","unstructured":"Jelinek, F.: Statistical Methods for Speech Recognition. The MIT Press, Cambridge (1997)"},{"key":"6_CR39","unstructured":"Jelinek, F., Mercer, M.: Interpolated estimation of Markov source parameters from sparse data. In: Proceedings of the Workshop on Pattern Recognition in Practice, pp. 381\u2013397. Amsterdam (1980)"},{"key":"6_CR40","doi-asserted-by":"publisher","first-page":"339","DOI":"10.1162\/tacl_a_00065","volume":"5","author":"M Johnson","year":"2017","unstructured":"Johnson, M., et al.: Google\u2019s multilingual neural machine translation system: enabling zero-shot translation. Trans. Assoc. Comput. Linguist. 5, 339\u2013351 (2017)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"6_CR41","unstructured":"Jurafsky, D., Martin, J.H.: Speech and Language Processing, 3\u00e9me \u00e9dition, 2018 edn. Prentice Hall (2000)"},{"key":"6_CR42","unstructured":"Kaplan, J., et al.: Scaling laws for neural language models (2020)"},{"key":"6_CR43","doi-asserted-by":"crossref","unstructured":"Karita, S., Soplin, N.E.Y., Watanabe, S., Delcroix, M., Ogawa, A., Nakatani, T.: Improving transformer-based end-to-end speech recognition with connectionist temporal classification and language model integration. In: Proceedings of the Interspeech 2019, pp. 1408\u20131412 (2019)","DOI":"10.21437\/Interspeech.2019-1938"},{"key":"6_CR44","doi-asserted-by":"publisher","unstructured":"Kudo, T.: Subword regularization: improving neural network translation models with multiple subword candidates. In: Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 66\u201375. Association for Computational Linguistics, Melbourne (2018). https:\/\/doi.org\/10.18653\/v1\/P18-1007","DOI":"10.18653\/v1\/P18-1007"},{"key":"6_CR45","doi-asserted-by":"crossref","unstructured":"Le, H.S., Oparin, I., Allauzen, A., Gauvain, J.L., Yvon, F.: Structured output layer neural network language model. In: Proceedings of IEEE International Conference on Acoustic, Speech and Signal Processing, Prague, Czech Republic, pp. 5524\u20135527 (2011)","DOI":"10.1109\/ICASSP.2011.5947610"},{"key":"6_CR46","unstructured":"Le, H., et al.: FlauBERT: unsupervised language model pre-training for french. In: LREC. Marseille, France (2020)"},{"key":"6_CR47","doi-asserted-by":"publisher","unstructured":"Lee, J.S., Hsiang, J.: Patent classification by fine-tuning BERT language model. World Patent Inf. 61, 101965 (2020). https:\/\/doi.org\/10.1016\/j.wpi.2020.101965","DOI":"10.1016\/j.wpi.2020.101965"},{"key":"6_CR48","doi-asserted-by":"publisher","unstructured":"Lee, J., et al.: BioBERT: a pre-trained biomedical language representation model for biomedical text mining. Bioinform. (Oxford Engl.) 36(4), 1234\u20131240 (2020). https:\/\/doi.org\/10.1093\/bioinformatics\/btz682","DOI":"10.1093\/bioinformatics\/btz682"},{"key":"6_CR49","doi-asserted-by":"publisher","unstructured":"Lewis, M., et al.: BART: denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7871\u20137880. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.703","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"6_CR50","doi-asserted-by":"publisher","first-page":"521","DOI":"10.1162\/tacl_a_00115","volume":"4","author":"T Linzen","year":"2016","unstructured":"Linzen, T., Dupoux, E., Goldberg, Y.: Assessing the ability of LSTMs to learn syntax-sensitive dependencies. Trans. Assoc. Comput. Linguist. 4, 521\u2013535 (2016)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"6_CR51","unstructured":"Liu, P.J., et al.: Generating Wikipedia by summarizing long sequences. CoRR abs\/1801.10198 (2018)"},{"key":"6_CR52","unstructured":"Liu, Y., et al.: RoBERTa: a robustly optimized BERT pretraining approach. CoRR abs\/1907.11692 (2019)"},{"key":"6_CR53","unstructured":"Lu, J., Batra, D., Parikh, D., Lee, S.: ViLBERT: pretraining task-agnostic visiolinguistic representations for vision-and-language tasks. In: Wallach, H., Larochelle, H., Beygelzimer, A., d\u2019Alch\u00e9-Buc, F., Fox, E., Garnett, R. (eds.) Advances in Neural Information Processing Systems, vol. 32. Curran Associates, Inc. (2019)"},{"key":"6_CR54","volume-title":"Foundations of Statistical Natural Language Processing","author":"CD Manning","year":"1999","unstructured":"Manning, C.D., Sch\u00fctze, H.: Foundations of Statistical Natural Language Processing. The MIT Press, Cambridge (1999)"},{"key":"6_CR55","doi-asserted-by":"publisher","unstructured":"Martin, L., et al.: CamemBERT: a tasty french language model. In: ACL 2020\u201358th Annual Meeting of the Association for Computational Linguistics. Seattle\/Virtual, United States (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.645","DOI":"10.18653\/v1\/2020.acl-main.645"},{"key":"6_CR56","doi-asserted-by":"publisher","unstructured":"Maruf, S., Saleh, F., Haffari, G.: A survey on document-level neural machine translation: Methods and evaluation. ACM Comput. Surv. 54(2) (2021). https:\/\/doi.org\/10.1145\/3441691","DOI":"10.1145\/3441691"},{"key":"6_CR57","doi-asserted-by":"publisher","unstructured":"Merrill, W., Weiss, G., Goldberg, Y., Schwartz, R., Smith, N.A., Yahav, E.: A formal hierarchy of RNN architectures. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 443\u2013459. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.43","DOI":"10.18653\/v1\/2020.acl-main.43"},{"key":"6_CR58","unstructured":"Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space. In: Proceedings of the International Conference on Representation Learning. ICLR (2013)"},{"key":"6_CR59","doi-asserted-by":"crossref","unstructured":"Mikolov, T., Karafi\u00e1t, M., Burget, L., \u010cernock\u00fd, J., Khudanpur, S.: Recurrent neural network based language model. In: Proceedings of the 11th Annual Conference of the International Speech Communication Association (Interspeech 2010), pp. 1045\u20131048. International Speech Communication Association (2010)","DOI":"10.21437\/Interspeech.2010-343"},{"key":"6_CR60","doi-asserted-by":"crossref","unstructured":"Peters, M., et al.: Deep contextualized word representations. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers), pp. 2227\u20132237. Association for Computational Linguistics, New Orleans (2018)","DOI":"10.18653\/v1\/N18-1202"},{"key":"6_CR61","doi-asserted-by":"publisher","unstructured":"Peters, M.E., et al.: Knowledge enhanced contextual word representations. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 43\u201354. Association for Computational Linguistics, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1005","DOI":"10.18653\/v1\/D19-1005"},{"key":"6_CR62","doi-asserted-by":"publisher","unstructured":"Petroni, F., et al.: Language models as knowledge bases? In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 2463\u20132473. Association for Computational Linguistics, Hong Kong (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1250","DOI":"10.18653\/v1\/D19-1250"},{"key":"6_CR63","unstructured":"Qi, D., Su, L., Song, J., Cui, E., Bharti, T., Sacheti, A.: ImageBERT: cross-modal pre-training with large-scale weak-supervised image-text data (2020)"},{"key":"6_CR64","unstructured":"Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., Sutskever, I.: Language models are unsupervised multitask learners. Technical report, OpenAI (2019)"},{"issue":"140","key":"6_CR65","first-page":"1","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res. 21(140), 1\u201367 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"6_CR66","doi-asserted-by":"publisher","unstructured":"Rajbhandari, S., Rasley, J., Ruwase, O., He, Y.: ZeRO: memory optimizations toward training trillion parameter models. In: SC20: International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201316 (2020). https:\/\/doi.org\/10.1109\/SC41405.2020.00024","DOI":"10.1109\/SC41405.2020.00024"},{"key":"6_CR67","doi-asserted-by":"publisher","first-page":"842","DOI":"10.1162\/tacl_a_00349","volume":"8","author":"A Rogers","year":"2020","unstructured":"Rogers, A., Kovaleva, O., Rumshisky, A.: A primer in BERTology: what we know about how BERT works. Trans. Assoc. Comput. Linguist. 8, 842\u2013866 (2020)","journal-title":"Trans. Assoc. Comput. Linguist."},{"issue":"8","key":"6_CR68","doi-asserted-by":"publisher","first-page":"1270","DOI":"10.1109\/5.880083","volume":"88","author":"R Rosenfeld","year":"2000","unstructured":"Rosenfeld, R.: Two decades of statistical language modeling: where do we go from here\u202f? Proc. IEEE 88(8), 1270\u20131278 (2000). https:\/\/doi.org\/10.1109\/5.880083","journal-title":"Proc. IEEE"},{"issue":"3","key":"6_CR69","doi-asserted-by":"publisher","first-page":"492","DOI":"10.1016\/j.csl.2006.09.003","volume":"21","author":"H Schwenk","year":"2007","unstructured":"Schwenk, H.: Continuous space language models. Comput. Speech Lang. 21(3), 492\u2013518 (2007)","journal-title":"Comput. Speech Lang."},{"key":"6_CR70","doi-asserted-by":"publisher","unstructured":"Sennrich, R., Haddow, B., Birch, A.: Neural machine translation of rare words with subword units. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Berlin, Germany, pp. 1715\u20131725 (2016). https:\/\/doi.org\/10.18653\/v1\/P16-1162","DOI":"10.18653\/v1\/P16-1162"},{"issue":"6","key":"6_CR71","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1145\/3347145","volume":"63","author":"NA Smith","year":"2020","unstructured":"Smith, N.A.: Contextual word representations: putting words into computers. Commun. ACM 63(6), 66\u201374 (2020). https:\/\/doi.org\/10.1145\/3347145","journal-title":"Commun. ACM"},{"key":"6_CR72","doi-asserted-by":"publisher","unstructured":"S\u00f8gaard, A., Vulic, I., Ruder, S., Faruqui, M.: Cross-lingual word embeddings. In: Synthesis Lectures on Human Language Technologies. Morgan & Claypool Publishers (2019). https:\/\/doi.org\/10.2200\/S00920ED2V01Y201904HLT042","DOI":"10.2200\/S00920ED2V01Y201904HLT042"},{"key":"6_CR73","doi-asserted-by":"publisher","unstructured":"Strubell, E., Ganesh, A., McCallum, A.: Energy and policy considerations for deep learning in NLP. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 3645\u20133650. Association for Computational Linguistics, Florence (2019). https:\/\/doi.org\/10.18653\/v1\/P19-1355","DOI":"10.18653\/v1\/P19-1355"},{"key":"6_CR74","doi-asserted-by":"publisher","unstructured":"Sun, C., Myers, A., Vondrick, C., Murphy, K., Schmid, C.: VideoBERT: a joint model for video and language representation learning. In: 2019 IEEE\/CVF International Conference on Computer Vision (ICCV), pp. 7463\u20137472 (2019). https:\/\/doi.org\/10.1109\/ICCV.2019.00756","DOI":"10.1109\/ICCV.2019.00756"},{"key":"6_CR75","unstructured":"Tay, Y., Dehghani, M., Bahri, D., Metzler, D.: Efficient transformers: a survey (2020)"},{"key":"6_CR76","unstructured":"Tiedemann, J.: Parallel data, tools and interfaces in opus. In: Choukri, K., et al. (eds.) Proceedings of the Eight International Conference on Language Resources and Evaluation (LREC 2012). European Language Resources Association (ELRA), Istanbul (2012)"},{"key":"6_CR77","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Guyon, I., et al. (eds.) Advances in Neural Information Processing Systems, vol. 30, pp. 5998\u20136008. Curran Associates, Inc. (2017)"},{"key":"6_CR78","unstructured":"de Vries, W., van Cranenburgh, A., Bisazza, A., Caselli, T., van Noord, G., Nissim, M.: Bertje: a dutch BERT model. CoRR abs\/1912.09582 (2019)"},{"key":"6_CR79","doi-asserted-by":"publisher","unstructured":"Wolf, T., et al.: Transformers: state-of-the-art natural language processing. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pp. 38\u201345. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-demos.6","DOI":"10.18653\/v1\/2020.emnlp-demos.6"}],"container-title":["Lecture Notes in Computer Science","Human-Centered Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-24349-3_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,17]],"date-time":"2024-10-17T15:57:54Z","timestamp":1729180674000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-24349-3_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031243486","9783031243493"],"references-count":79,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-24349-3_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"4 April 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ACAI","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ECCAI Advanced Course on Artificial Intelligence","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Berlin","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 October 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 October 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"acai-agents2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.humane-ai.eu\/event\/acai2021\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"easychair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"21","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"21","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"100% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}