{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:45:29Z","timestamp":1784137529598,"version":"3.55.0"},"reference-count":87,"publisher":"MIT Press","license":[{"start":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T00:00:00Z","timestamp":1706832000000},"content-version":"vor","delay-in-days":32,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,31]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>This paper introduces mGPT, a multilingual variant of GPT-3, pretrained on 61 languages from 25 linguistically diverse language families using Wikipedia and the C4 Corpus. We detail the design and pretraining procedure. The models undergo an intrinsic and extrinsic evaluation: language modeling in all languages, downstream evaluation on cross-lingual NLU datasets and benchmarks in 33 languages, and world knowledge probing in 23 languages. The in-context learning abilities are on par with the contemporaneous language models while covering a larger number of languages, including underrepresented and low-resource languages of the Commonwealth of Independent States and the indigenous peoples in Russia. The source code and the language models are publicly available under the MIT license.<\/jats:p>","DOI":"10.1162\/tacl_a_00633","type":"journal-article","created":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T23:51:18Z","timestamp":1706917878000},"page":"58-79","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":35,"title":["mGPT: Few-Shot Learners Go Multilingual"],"prefix":"10.1162","volume":"12","author":[{"given":"Oleh","family":"Shliazhko","sequence":"first","affiliation":[{"name":"Independent Researcher, The Netherlands. olehshliazhko@gmail.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alena","family":"Fenogenova","sequence":"additional","affiliation":[{"name":"SaluteDevices, Russia. alenush93@gmail.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maria","family":"Tikhonova","sequence":"additional","affiliation":[{"name":"Independent Researcher, The Netherlands"},{"name":"SaluteDevices, Russia. mtihonova@hse.ru"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anastasia","family":"Kozlova","sequence":"additional","affiliation":[{"name":"SaluteDevices, Russia. anastasi2510@gmail.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vladislav","family":"Mikhailov","sequence":"additional","affiliation":[{"name":"SaluteDevices, Russia. vvmkhlvv@gmail.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tatiana","family":"Shavrina","sequence":"additional","affiliation":[{"name":"SaluteDevices, Russia"},{"name":"AIRI, Russia"},{"name":"AI Center, NUST MISiS, Russia"},{"name":"Institute of Linguistics RAS, Russia. rybolos@gmail.com"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"281","published-online":{"date-parts":[[2024,1,31]]},"reference":[{"key":"2024020223510424600_bib1","doi-asserted-by":"publisher","first-page":"100","DOI":"10.18653\/v1\/2022.mrl-1.10","article-title":"Transformers on multilingual clause-level morphology","volume-title":"Proceedings of the The 2nd Workshop on Multi-lingual Representation Learning (MRL)","author":"Acikgoz","year":"2022"},{"key":"2024020223510424600_bib2","doi-asserted-by":"publisher","first-page":"5454","DOI":"10.18653\/v1\/2022.acl-long.374","article-title":"Multi task learning for zero shot performance prediction of multilingual models","volume-title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Ahuja","year":"2022"},{"key":"2024020223510424600_bib3","doi-asserted-by":"publisher","first-page":"89","DOI":"10.18653\/v1\/W19-3712","article-title":"Tuning multilingual transformers for language-specific named entity recognition","volume-title":"Proceedings of the 7th Workshop on Balto-Slavic Natural Language Processing","author":"Arkhipov","year":"2019"},{"key":"2024020223510424600_bib4","unstructured":"Giusepppe\n              Attardi\n            \n          . 2015. WikiExtractor. https:\/\/github.com\/attardi\/wikiextractor"},{"key":"2024020223510424600_bib5","doi-asserted-by":"publisher","first-page":"610","DOI":"10.1145\/3442188.3445922","article-title":"On the dangers of stochastic parrots: Can language models be too big?","volume-title":"Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency","author":"Bender","year":"2021"},{"key":"2024020223510424600_bib6","first-page":"2397","article-title":"Pythia: A suite for analyzing large language models across training and scaling","volume-title":"International Conference on Machine Learning","author":"Biderman","year":"2023"},{"key":"2024020223510424600_bib7","doi-asserted-by":"publisher","first-page":"95","DOI":"10.18653\/v1\/2022.bigscience-1.9","article-title":"GPT-NeoX-20B: An open- source autoregressive language model","volume-title":"Proceedings of BigScience Episode #5 \u2013 Workshop on Challenges & Perspectives in Creating Large Language Models","author":"Black","year":"2022"},{"key":"2024020223510424600_bib8","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Advances in Neural Information Processing Systems","author":"Brown","year":"2020"},{"key":"2024020223510424600_bib9","doi-asserted-by":"publisher","first-page":"2508","DOI":"10.18653\/v1\/2022.findings-naacl.192","article-title":"MTG: A benchmark suite for multilingual text generation","volume-title":"Findings of the Association for Computational Linguistics: NAACL 2022","author":"Chen","year":"2022"},{"key":"2024020223510424600_bib10","article-title":"Generating long sequences with sparse transformers","author":"Child","year":"2019"},{"key":"2024020223510424600_bib11","article-title":"Rethinking embedding coupling in pre-trained language models","volume-title":"International Conference on Learning Representations","author":"Chung","year":"2021"},{"key":"2024020223510424600_bib12","doi-asserted-by":"publisher","first-page":"8440","DOI":"10.18653\/v1\/2020.acl-main.747","article-title":"Unsupervised cross-lingual representation learning at scale","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Conneau","year":"2020"},{"key":"2024020223510424600_bib13","doi-asserted-by":"publisher","first-page":"2475","DOI":"10.18653\/v1\/D18-1269","article-title":"XNLI: Evaluating cross-lingual sentence representations","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Conneau","year":"2018"},{"key":"2024020223510424600_bib14","doi-asserted-by":"publisher","first-page":"536","DOI":"10.18653\/v1\/N18-2085","article-title":"Are all languages equally hard to language-model?","volume-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)","author":"Cotterell","year":"2018"},{"key":"2024020223510424600_bib15","doi-asserted-by":"publisher","first-page":"512","DOI":"10.1609\/icwsm.v11i1.14955","article-title":"Automated hate speech detection and the problem of offensive language","volume-title":"Proceedings of the International AAAI Conference on Web and Social Media","author":"Davidson","year":"2017"},{"key":"2024020223510424600_bib16","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","author":"Devlin","year":"2019"},{"key":"2024020223510424600_bib17","unstructured":"Nolan\n              Dey\n            , GurpreetGosal, Zhiming, Chen, HemantKhachane, WilliamMarshall, RibhuPathria, MarvinTom, and JoelHestness. 2023. Cerebras-GPT: Open compute-optimal language models trained on the cerebras wafer-scale cluster. 10.48550\/arXiv.2304.03208"},{"key":"2024020223510424600_bib18","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2002.06305","article-title":"Fine-tuning pretrained language models: Weight initializations, data orders, and early stopping","author":"Dodge","year":"2020"},{"key":"2024020223510424600_bib19","first-page":"564","article-title":"Do we name the languages we study? The #BenderRule in LREC and ACL articles","volume-title":"Proceedings of the Thirteenth Language Resources and Evaluation Conference","author":"Ducel","year":"2022"},{"key":"2024020223510424600_bib20","doi-asserted-by":"publisher","first-page":"1131","DOI":"10.1613\/jair.1.12918","article-title":"Neural natural language generation: A survey on multilinguality, multimodality, controllability and learning","volume":"73","author":"Erdem","year":"2022","journal-title":"Journal of Artificial Intelligence Research"},{"key":"2024020223510424600_bib21","doi-asserted-by":"publisher","first-page":"96","DOI":"10.18653\/v1\/2021.gem-1.10","article-title":"The GEM benchmark: Natural language generation, its evaluation and metrics","volume-title":"Proceedings of the 1st Workshop on Natural Language Generation, Evaluation, and Metrics (GEM 2021)","author":"Gehrmann","year":"2021"},{"key":"2024020223510424600_bib22","doi-asserted-by":"publisher","first-page":"134","DOI":"10.18653\/v1\/2022.mrl-1.14","article-title":"The MRL 2022 shared task on multilingual clause-level morphology","volume-title":"Proceedings of the The 2nd Workshop on Multi-lingual Representation Learning (MRL)","author":"Goldman","year":"2022"},{"key":"2024020223510424600_bib23","doi-asserted-by":"publisher","first-page":"2545","DOI":"10.18653\/v1\/2021.naacl-main.201","article-title":"A survey on recent approaches for natural language processing in low-resource scenarios","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Hedderich","year":"2021"},{"key":"2024020223510424600_bib24","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2203.15556","article-title":"Training compute-optimal large language models","author":"Hoffmann","year":"2022"},{"key":"2024020223510424600_bib25","first-page":"4411","article-title":"XTREME: A massively multilingual multi-task benchmark for evaluating cross-lingual generalisation","volume-title":"International Conference on Machine Learning","author":"Junjie","year":"2020"},{"key":"2024020223510424600_bib26","doi-asserted-by":"publisher","first-page":"2485","DOI":"10.18653\/v1\/D19-1252","article-title":"Unicoder: A universal language encoder by pre-training with multiple cross-lingual tasks","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Huang","year":"2019"},{"key":"2024020223510424600_bib27","doi-asserted-by":"publisher","first-page":"2296","DOI":"10.18653\/v1\/2020.coling-main.208","article-title":"Automatic detection of machine generated text: A critical survey","volume-title":"Proceedings of the 28th International Conference on Computational Linguistics","author":"Jawahar","year":"2020"},{"key":"2024020223510424600_bib28","doi-asserted-by":"publisher","first-page":"4948","DOI":"10.18653\/v1\/2020.findings-emnlp.445","article-title":"IndicNLPSuite: Monolingual corpora, evaluation benchmarks and pre-trained multilingual language models for Indian languages","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Kakwani","year":"2020"},{"key":"2024020223510424600_bib29","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2108.05542","article-title":"AMMUS: A survey of transformer-based pretrained models in natural language processing","author":"Kalyan","year":"2021"},{"key":"2024020223510424600_bib30","doi-asserted-by":"publisher","first-page":"3250","DOI":"10.18653\/v1\/2021.eacl-main.284","article-title":"Multilingual LAMA: Investigating knowledge in multilingual pretrained language models","volume-title":"Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume","author":"Kassner","year":"2021"},{"key":"2024020223510424600_bib31","first-page":"28","article-title":"Eastern Armenian national corpus: State of the art and perspectives","volume-title":"Proceedings of the Workshop on Processing Language Variation: Digital Armenian (DigitAm) within the 13th Language Resources and Evaluation Conference","author":"Khurshudyan","year":"2022"},{"key":"2024020223510424600_bib32","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1162\/tacl_a_00447","article-title":"Quality at a glance: An audit of web-crawled multilingual datasets","volume":"10","author":"Kreutzer","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024020223510424600_bib33","doi-asserted-by":"publisher","first-page":"4483","DOI":"10.18653\/v1\/2020.emnlp-main.363","article-title":"From zero to hero: On the limitations of zero-shot language transfer with multilingual Transformers","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Lauscher","year":"2020"},{"key":"2024020223510424600_bib34","doi-asserted-by":"publisher","first-page":"6008","DOI":"10.18653\/v1\/2020.emnlp-main.484","article-title":"XGLUE: A new benchmark datasetfor cross-lingual pre-training, understanding and generation","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Liang","year":"2020"},{"key":"2024020223510424600_bib35","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2112.10668","article-title":"Few-shot learning with multilingual language models","author":"Xi","year":"2022"},{"key":"2024020223510424600_bib36","first-page":"923","article-title":"OpenSubtitles2016: Extracting large parallel corpora from movie and TV subtitles","volume-title":"Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916)","author":"Lison","year":"2016"},{"issue":"9","key":"2024020223510424600_bib37","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3560815","article-title":"Pre-Train, prompt, and predict: A systematic survey of prompting methods in natural language processing","volume":"55","author":"Liu","year":"2023","journal-title":"ACM Computing Surveys"},{"key":"2024020223510424600_bib38","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2003.07278","article-title":"A Survey on Contextual Embeddings","author":"Qi","year":"2020"},{"key":"2024020223510424600_bib39","doi-asserted-by":"publisher","first-page":"993","DOI":"10.18653\/v1\/2022.findings-emnlp.70","article-title":"Autoregressive structured prediction with language models","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2022","author":"Liu","year":"2022"},{"key":"2024020223510424600_bib40","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2023.08.012","article-title":"GPT understands, too","author":"Liu","year":"2023","journal-title":"AI Open"},{"key":"2024020223510424600_bib41","doi-asserted-by":"publisher","first-page":"726","DOI":"10.1162\/tacl_a_00343","article-title":"Multilingual denoising pre-training for neural machine translation","volume":"8","author":"Liu","year":"2020","journal-title":"Transactions of the Association for Computational Linguistics"},{"issue":"1","key":"2024020223510424600_bib42","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1214\/aoms\/1177730491","article-title":"Controlling the false discovery rate: A practical and powerful approach to multiple testing","volume":"18","author":"Mann","year":"1947","journal-title":"Annals of Mathematical Statistics"},{"key":"2024020223510424600_bib43","doi-asserted-by":"publisher","first-page":"7203","DOI":"10.18653\/v1\/2020.acl-main.645","article-title":"CamemBERT: A tasty French language model","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Martin","year":"2020"},{"key":"2024020223510424600_bib44","doi-asserted-by":"publisher","first-page":"6626","DOI":"10.18653\/v1\/2020.coling-main.581","article-title":"RoBERT \u2013 a Romanian BERT model","volume-title":"Proceedings of the 28th International Conference on Computational Linguistics","author":"Masala","year":"2020"},{"key":"2024020223510424600_bib45","doi-asserted-by":"publisher","first-page":"217","DOI":"10.18653\/v1\/2020.blackboxnlp-1.21","article-title":"BERTs of a feather do not generalize together: Large variability in generalization across models with similar test set performance","volume-title":"Proceedings of the Third BlackboxNLP Workshop on Analyzing and Interpreting Neural Networks for NLP","author":"Thomas McCoy","year":"2020"},{"key":"2024020223510424600_bib46","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2112.10508","article-title":"Between words and characters: A brief history of open-vocabulary modeling and tokenization in NLP","author":"Mielke","year":"2021"},{"key":"2024020223510424600_bib47","first-page":"2340","article-title":"Stress test evaluation for natural language inference","volume-title":"Proceedings of the 27th International Conference on Computational Linguistics","author":"Naik","year":"2018"},{"key":"2024020223510424600_bib48","doi-asserted-by":"publisher","first-page":"4658","DOI":"10.18653\/v1\/P19-1459","article-title":"Probing neural network comprehension of natural language arguments","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Niven","year":"2019"},{"key":"2024020223510424600_bib49","first-page":"1659","article-title":"Universal Dependencies v1: A multilingual treebank collection","volume-title":"Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916)","author":"Nivre","year":"2016"},{"key":"2024020223510424600_bib50","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2102.13019","article-title":"Investigating the limitations of transformers with simple arithmetic tasks","author":"Nogueira","year":"2021"},{"key":"2024020223510424600_bib51","first-page":"498","article-title":"Russian minority languages on the web: Descriptive statistics","volume-title":"Vladimir Selegey (chief ed.), Computational Linguistics and Intellectual Technologies: Proceedings of the International Conference \u201cDialogue\u201d","author":"Orekhov","year":"2016"},{"key":"2024020223510424600_bib52","doi-asserted-by":"publisher","first-page":"9","DOI":"10.14618\/ids-pub-9021","article-title":"Asynchronous pipelines for processing huge corpora on medium to low resource infrastructures","volume-title":"Proceedings of the Workshop on Challenges in the Management of Large Corpora (CMLC-7) 2019. Cardiff, 22nd July 2019","author":"Su\u00e1rez","year":"2019"},{"key":"2024020223510424600_bib53","doi-asserted-by":"publisher","first-page":"48","DOI":"10.18653\/v1\/N19-4009","article-title":"fairseq: A fast, extensible toolkit for sequence modeling","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations)","author":"Ott","year":"2019"},{"key":"2024020223510424600_bib54","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/978-1-4842-7341-8_3","article-title":"TensorFlow datasets","author":"Paper","year":"2021","journal-title":"State-of-the-Art Deep Learning Models in TensorFlow: Modern Machine Learning in the Google Colab Ecosystem"},{"issue":"347-352","key":"2024020223510424600_bib55","doi-asserted-by":"publisher","first-page":"240","DOI":"10.1098\/rspl.1895.0041","article-title":"Note on regression and inheritance in the case of two parents","volume":"58","author":"Pearson","year":"1895","journal-title":"Proceedings of the Royal Society of London"},{"key":"2024020223510424600_bib56","article-title":"True few-shot learning with language models","volume":"34","author":"Perez","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024020223510424600_bib57","doi-asserted-by":"publisher","first-page":"2463","DOI":"10.18653\/v1\/D19-1250","article-title":"Language models as knowledge bases?","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Petroni","year":"2019"},{"key":"2024020223510424600_bib58","doi-asserted-by":"publisher","first-page":"10186","DOI":"10.18653\/v1\/2021.emnlp-main.800","article-title":"UNKs everywhere: Adapting multilingual language models to new scripts","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Pfeiffer","year":"2021"},{"key":"2024020223510424600_bib59","doi-asserted-by":"publisher","first-page":"2362","DOI":"10.18653\/v1\/2020.emnlp-main.185","article-title":"XCOPA: A multilingual dataset for causal commonsense reasoning","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Ponti","year":"2020"},{"issue":"8","key":"2024020223510424600_bib60","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"2024020223510424600_bib61","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"Journal of Machine Learning Research"},{"key":"2024020223510424600_bib62","doi-asserted-by":"publisher","first-page":"3505","DOI":"10.1145\/3394486.3406703","article-title":"DeepSpeed: System optimizations enable training deep learning models with over 100 billion parameters","volume-title":"Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","author":"Rasley","year":"2020"},{"key":"2024020223510424600_bib63","doi-asserted-by":"publisher","first-page":"3118","DOI":"10.18653\/v1\/2021.acl-long.243","article-title":"How good is your tokenizer? On the monolingual performance of multilingual language models","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)","author":"Rust","year":"2021"},{"key":"2024020223510424600_bib64","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2211.05100","article-title":"BLOOM: A 176B-parameter open-access multilingual language model","author":"Scao","year":"2023"},{"key":"2024020223510424600_bib65","doi-asserted-by":"publisher","first-page":"2339","DOI":"10.18653\/v1\/2021.naacl-main.185","article-title":"It\u2019s not just size that matters: Small language models are also few-shot learners","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Schick","year":"2021"},{"key":"2024020223510424600_bib66","unstructured":"Mohammad\n              Shoeybi\n            , MostofaPatwary, RaulPuri, PatrickLeGresley, JaredCasper, and BryanCatanzaro. 2020. Megatron-LM: Training multi-billion parameter language models using model parallelism. 10.48550\/arXiv.1909.08053"},{"key":"2024020223510424600_bib67","doi-asserted-by":"publisher","first-page":"3645","DOI":"10.18653\/v1\/P19-1355","article-title":"Energy and policy considerations for deep learning in NLP","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Strubell","year":"2019"},{"key":"2024020223510424600_bib68","doi-asserted-by":"publisher","first-page":"3534","DOI":"10.18653\/v1\/2021.findings-acl.310","article-title":"It\u2019s all in the heads: Using attention heads as a baseline for cross-lingual transfer in commonsense reasoning","volume-title":"Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021","author":"Tikhonov","year":"2021"},{"key":"2024020223510424600_bib69","article-title":"Attention is all you need","volume":"30","author":"Vaswani","year":"2017","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024020223510424600_bib70","article-title":"Multitask prompted training enables zero-shot task generalization","volume-title":"International Conference on Learning Representations","author":"Victor","year":"2022"},{"key":"2024020223510424600_bib71","article-title":"SuperGLUE: A stickier benchmark for general-purpose language understanding systems","volume":"32","author":"Wang","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024020223510424600_bib72","article-title":"GPT-J-6B: A 6 billion parameter autoregressive language model","author":"Wang","year":"2021"},{"key":"2024020223510424600_bib73","doi-asserted-by":"publisher","first-page":"9154","DOI":"10.1609\/aaai.v34i05.6451","article-title":"Neural machine translation with byte-level subwords","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Wang","year":"2020"},{"key":"2024020223510424600_bib74","doi-asserted-by":"publisher","first-page":"4195","DOI":"10.18653\/v1\/2021.findings-emnlp.354","article-title":"Want to reduce labeling cost? GPT-3 can help","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2021","author":"Wang","year":"2021"},{"key":"2024020223510424600_bib75","doi-asserted-by":"publisher","first-page":"214","DOI":"10.1145\/3531146.3533088","article-title":"Taxonomy of risks posed by language models","volume-title":"Proceedings of the 2022 ACM Conference on Fairness, Accountability, and Transparency","author":"Weidinger","year":"2022"},{"key":"2024020223510424600_bib76","first-page":"4003","article-title":"CCNet: Extracting high quality monolingual datasets from web crawl data","volume-title":"Proceedings of the 12th Language Resources and Evaluation Conference","author":"Wenzek","year":"2020"},{"key":"2024020223510424600_bib77","doi-asserted-by":"publisher","first-page":"1","DOI":"10.18653\/v1\/2021.mrl-1.1","article-title":"Language models are few-shot multilingual learners","volume-title":"Proceedings of the 1st Workshop on Multilingual Representation Learning","author":"Winata","year":"2021"},{"key":"2024020223510424600_bib78","doi-asserted-by":"publisher","first-page":"38","DOI":"10.18653\/v1\/2020.emnlp-demos.6","article-title":"Transformers: State-of-the-art natural language processing","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations","author":"Wolf","year":"2020"},{"key":"2024020223510424600_bib79","doi-asserted-by":"publisher","first-page":"120","DOI":"10.18653\/v1\/2020.repl4nlp-1.16","article-title":"Are all languages created equal in multilingual BERT?","volume-title":"Proceedings of the 5th Workshop on Representation Learning for NLP","author":"Shijie","year":"2020"},{"key":"2024020223510424600_bib80","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.10429","article-title":"DoReMi: Optimizing data mixtures speeds up language model pretraining","author":"Xie","year":"2023"},{"key":"2024020223510424600_bib81","doi-asserted-by":"publisher","first-page":"483","DOI":"10.18653\/v1\/2021.naacl-main.41","article-title":"mT5: A massively multilingual pre-trained text-to-text transformer","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Xue","year":"2021"},{"key":"2024020223510424600_bib82","doi-asserted-by":"publisher","first-page":"3687","DOI":"10.18653\/v1\/D19-1382","article-title":"PAWS-X: A cross-lingual adversarial dataset for paraphrase identification","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Yang","year":"2019"},{"key":"2024020223510424600_bib83","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume":"32","author":"Yang","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024020223510424600_bib84","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2104.12369","article-title":"PanGu-\u03b1: Large-scale autoregressive pretrained chinese language models with auto-parallel computation","author":"Zeng","year":"2021"},{"key":"2024020223510424600_bib85","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2205.01068","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022"},{"key":"2024020223510424600_bib86","first-page":"12697","article-title":"Calibrate before use: Improving few-shot performance of language models","volume-title":"International Conference on Machine Learning","author":"Zhao","year":"2021"},{"key":"2024020223510424600_bib87","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2309.10931","article-title":"A family of pretrained transformer language models for Russian","author":"Zmitrovich","year":"2023"}],"container-title":["Transactions of the Association for Computational Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00633\/2325676\/tacl_a_00633.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00633\/2325676\/tacl_a_00633.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T23:51:48Z","timestamp":1706917908000},"score":1,"resource":{"primary":{"URL":"https:\/\/direct.mit.edu\/tacl\/article\/doi\/10.1162\/tacl_a_00633\/119278\/mGPT-Few-Shot-Learners-Go-Multilingual"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":87,"URL":"https:\/\/doi.org\/10.1162\/tacl_a_00633","relation":{},"ISSN":["2307-387X"],"issn-type":[{"value":"2307-387X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024]]},"published":{"date-parts":[[2024]]}}}