{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T02:29:14Z","timestamp":1793500154063,"version":"4.5.0"},"reference-count":60,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Neurocomputing"],"published-print":{"date-parts":[[2024,2]]},"DOI":"10.1016\/j.neucom.2023.127063","type":"journal-article","created":{"date-parts":[[2023,11,24]],"date-time":"2023-11-24T07:28:27Z","timestamp":1700810907000},"page":"127063","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1901,"special_numbering":"C","title":["RoFormer: Enhanced transformer with Rotary Position Embedding"],"prefix":"10.1016","volume":"568","author":[{"given":"Jianlin","family":"Su","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0741-0710","authenticated-orcid":false,"given":"Murtadha","family":"Ahmed","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shengfeng","family":"Pan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wen","family":"Bo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yunfeng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.neucom.2023.127063_b1","series-title":"Proceedings of the 34th International Conference on Machine Learning, ICML 2017, Sydney, NSW, Australia, 6-11 August 2017","first-page":"1243","article-title":"Convolutional sequence to sequence learning","volume":"vol. 70","author":"Gehring","year":"2017"},{"key":"10.1016\/j.neucom.2023.127063_b2","series-title":"8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020","article-title":"How much position information do convolutional neural networks encode?","author":"Islam","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b3","doi-asserted-by":"crossref","first-page":"771","DOI":"10.1162\/tacl_a_00574","article-title":"Rank-Aware Negative Training for Semi-Supervised Text Classification","volume":"11","author":"Murtadha","year":"2023","journal-title":"Transactions of the Association for Computational Linguistics","ISSN":"https:\/\/id.crossref.org\/issn\/2307-387X","issn-type":"print"},{"key":"10.1016\/j.neucom.2023.127063_b4","series-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers)","first-page":"4171","article-title":"BERT: pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b5","series-title":"Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4-9, 2017, Long Beach, CA, USA","first-page":"5998","article-title":"Attention is all you need","author":"Vaswani","year":"2017"},{"key":"10.1016\/j.neucom.2023.127063_b6","series-title":"Language models are unsupervised multitask learners","author":"Radford","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b7","series-title":"8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020","article-title":"Are transformers universal approximators of sequence-to-sequence functions?","author":"Yun","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b8","series-title":"8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020","article-title":"ALBERT: a lite BERT for self-supervised learning of language representations","author":"Lan","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b9","series-title":"8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020","article-title":"ELECTRA: pre-training text encoders as discriminators rather than generators","author":"Clark","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b10","series-title":"Improving Language Understanding by Generative Pre-Training","author":"Radford","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b11","series-title":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, EMNLP 2016, Austin, Texas, USA, November 1-4, 2016","first-page":"2249","article-title":"A decomposable attention model for natural language inference","author":"Parikh","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b12","series-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT, New Orleans, Louisiana, USA, June 1-6, 2018, Volume 2 (Short Papers)","first-page":"464","article-title":"Self-attention with relative position representations","author":"Shaw","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b13","series-title":"7th International Conference on Learning Representations, ICLR 2019, New Orleans, la, USA, May 6-9, 2019","article-title":"Music transformer: Generating music with long-term structure","author":"Huang","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b14","series-title":"Proceedings of the 57th Conference of the Association for Computational Linguistics, ACL 2019, Florence, Italy, July 28- August 2, 2019, Volume 1: Long Papers","first-page":"2978","article-title":"Transformer-XL: Attentive language models beyond a fixed-length context","author":"Dai","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b15","series-title":"Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, December 8-14, 2019, Vancouver, BC, Canada","first-page":"5754","article-title":"Xlnet: Generalized autoregressive pretraining for language understanding","author":"Yang","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b16","first-page":"140:1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.neucom.2023.127063_b17","series-title":"9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021","article-title":"Rethinking positional encoding in language pre-training","author":"Ke","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b18","series-title":"9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021","article-title":"Deberta: decoding-enhanced bert with disentangled attention","author":"He","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b19","series-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","first-page":"3327","article-title":"Improve transformer models with better relative position embeddings","author":"Huang","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b20","series-title":"Proceedings of the 37th International Conference on Machine Learning, ICML 2020, 13-18 July 2020, Virtual Event","first-page":"6327","article-title":"Learning to encode position for transformer with continuous dynamical model","volume":"vol. 119","author":"Liu","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b21","series-title":"Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018, NeurIPS 2018, December 3-8, 2018, Montr\u00c9Al, Canada","first-page":"6572","article-title":"Neural ordinary differential equations","author":"Chen","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b22","series-title":"8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020","article-title":"Encoding word order in complex embeddings","author":"Wang","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b23","series-title":"Proceedings of the 37th International Conference on Machine Learning, ICML 2020, 13-18 July 2020, Virtual Event","first-page":"5156","article-title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","volume":"vol. 119","author":"Katharopoulos","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b24","series-title":"IEEE Winter Conference on Applications of Computer Vision, WACV 2021, Waikoloa, HI, USA, January 3-8, 2021","first-page":"3530","article-title":"Efficient attention: Attention with linear complexities","author":"Shen","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b25","series-title":"7th International Conference on Learning Representations, ICLR 2019, New Orleans, la, USA, May 6-9, 2019","article-title":"GLUE: a multi-task benchmark and analysis platform for natural language understanding","author":"Wang","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b26","series-title":"9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021","article-title":"Rethinking attention with performers","author":"Choromanski","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b27","series-title":"Proceedings of the Ninth Workshop on Statistical Machine Translation, WMT@ACL 2014, June 26-27, 2014, Baltimore, Maryland, USA","first-page":"12","article-title":"Findings of the 2014 workshop on statistical machine translation","author":"Bojar","year":"2014"},{"key":"10.1016\/j.neucom.2023.127063_b28","series-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics, ACL 2016, August 7-12, 2016, Berlin, Germany, Volume 1: Long Papers","article-title":"Neural machine translation of rare words with subword units","author":"Sennrich","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b29","series-title":"Proceedings of the 13th International Conference on Spoken Language Translation, IWSLT 2016, Seattle, WA, USA, December 8-9, 2016","article-title":"Is neural machine translation ready for deployment? A case study on 30 translation directions","author":"Junczys-Dowmunt","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b30","doi-asserted-by":"crossref","first-page":"43","DOI":"10.2478\/pralin-2018-0002","article-title":"Training tips for the transformer model","volume":"110","author":"Popel","year":"2018","journal-title":"Prague Bull. Math. Linguist."},{"key":"10.1016\/j.neucom.2023.127063_b31","series-title":"Google\u2019s neural machine translation system: Bridging the gap between human and machine translation","author":"Wu","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b32","series-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT, New Orleans, Louisiana, USA, June 1-6, 2018, Volume 2 (Short Papers)","first-page":"464","article-title":"Self-attention with relative position representations","author":"Shaw","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b33","series-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Demonstrations","first-page":"48","article-title":"Fairseq: A fast, extensible toolkit for sequence modeling","author":"Ott","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b34","series-title":"Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics, July 6-12, 2002, Philadelphia, PA, USA","first-page":"311","article-title":"Bleu: a method for automatic evaluation of machine translation","author":"Papineni","year":"2002"},{"key":"10.1016\/j.neucom.2023.127063_b35","series-title":"2015 IEEE International Conference on Computer Vision, ICCV 2015, Santiago, Chile, December 7-13, 2015","first-page":"19","article-title":"Aligning books and movies: Towards story-like visual explanations by watching movies and reading books","author":"Zhu","year":"2015"},{"key":"10.1016\/j.neucom.2023.127063_b36","series-title":"Wikimedia downloads","author":"Foundation","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b37","series-title":"7th International Conference on Learning Representations, ICLR 2019, New Orleans, la, USA, May 6-9, 2019","article-title":"Decoupled weight decay regularization","author":"Loshchilov","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b38","series-title":"Proceedings of the Third International Workshop on Paraphrasing, IWP@IJCNLP 2005, Jeju Island, Korea, October 2005, 2005","article-title":"Automatically constructing a corpus of sentential paraphrases","author":"Dolan","year":"2005"},{"key":"10.1016\/j.neucom.2023.127063_b39","series-title":"Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, EMNLP 2013, 18-21 October 2013, Grand Hyatt Seattle, Seattle, Washington, USA, a Meeting of SIGDAT, a Special Interest Group of the ACL","first-page":"1631","article-title":"Recursive deep models for semantic compositionality over a sentiment treebank","author":"Socher","year":"2013"},{"key":"10.1016\/j.neucom.2023.127063_b40","series-title":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, EMNLP 2016, Austin, Texas, USA, November 1-4, 2016","first-page":"2383","article-title":"Squad: 100, 000+ questions for machine comprehension of text","author":"Rajpurkar","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b41","series-title":"Proceedings of the 11th International Workshop on Semantic Evaluation, SemEval@ACL 2017, Vancouver, Canada, August 3-4, 2017","first-page":"115","article-title":"Udl at SemEval-2017 task 1: Semantic textual similarity estimation of english sentence pairs using regression model over pairwise features","author":"Al-Natsheh","year":"2017"},{"key":"10.1016\/j.neucom.2023.127063_b42","series-title":"Quora question pairs","author":"Chen","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b43","series-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2018, New Orleans, Louisiana, USA, June 1-6, 2018, Volume 1 (Long Papers)","first-page":"1112","article-title":"A broad-coverage challenge corpus for sentence understanding through inference","author":"Williams","year":"2018"},{"key":"10.1016\/j.neucom.2023.127063_b44","series-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations","first-page":"38","article-title":"Transformers: State-of-the-art natural language processing","author":"Wolf","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b45","series-title":"Large text compression benchmark","author":"Mahoney","year":"2011"},{"key":"10.1016\/j.neucom.2023.127063_b46","unstructured":"J. Su, WoBERT: Word-based Chinese BERT model - ZhuiyiAI, Tech. rep., 2020, URL."},{"key":"10.1016\/j.neucom.2023.127063_b47","series-title":"NEZHA: neural contextualized representation for Chinese language understanding","author":"Wei","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b48","series-title":"CAIL2019-SCM: a dataset of similar case matching in legal domain","author":"Xiao","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b49","series-title":"Rotary embeddings: A relative revolution","author":"Biderman","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b50","series-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, Virtual","article-title":"Language models are few-shot learners","author":"Brown","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b51","first-page":"140:1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.neucom.2023.127063_b52","series-title":"The pile: An 800gb dataset of diverse text for language modeling","author":"Gao","year":"2021"},{"key":"10.1016\/j.neucom.2023.127063_b53","series-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics, ACL 2016, August 7-12, 2016, Berlin, Germany, Volume 1: Long Papers","article-title":"The LAMBADA dataset: Word prediction requiring a broad discourse context","author":"Paperno","year":"2016"},{"key":"10.1016\/j.neucom.2023.127063_b54","series-title":"Proceedings of the 57th Conference of the Association for Computational Linguistics, ACL 2019, Florence, Italy, July 28- August 2, 2019, Volume 1: Long Papers","first-page":"4791","article-title":"HellaSwag: Can a machine really finish your sentence?","author":"Zellers","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b55","series-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics, ACL 2017, Vancouver, Canada, July 30 - August 4, Volume 1: Long Papers","first-page":"1601","article-title":"Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension","author":"Joshi","year":"2017"},{"key":"10.1016\/j.neucom.2023.127063_b56","series-title":"Amos: An adam-style optimizer with adaptive weight decay towards model-oriented scale","author":"Tian","year":"2022"},{"key":"10.1016\/j.neucom.2023.127063_b57","series-title":"End-to-end object detection with transformers","author":"Carion","year":"2020"},{"key":"10.1016\/j.neucom.2023.127063_b58","series-title":"2015 IEEE International Conference on Computer Vision, ICCV 2015, Santiago, Chile, December 7-13, 2015","first-page":"19","article-title":"Aligning books and movies: Towards story-like visual explanations by watching movies and reading books","author":"Zhu","year":"2015"},{"key":"10.1016\/j.neucom.2023.127063_b59","series-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"Liu","year":"2019"},{"key":"10.1016\/j.neucom.2023.127063_b60","series-title":"The Tenth International Conference on Learning Representations, ICLR 2022, Virtual Event, April 25-29, 2022","article-title":"Train short, test long: Attention with linear biases enables input length extrapolation","author":"Press","year":"2022"}],"container-title":["Neurocomputing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0925231223011864?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0925231223011864?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,13]],"date-time":"2025-10-13T21:28:49Z","timestamp":1760390929000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0925231223011864"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2]]},"references-count":60,"alternative-id":["S0925231223011864"],"URL":"https:\/\/doi.org\/10.1016\/j.neucom.2023.127063","relation":{},"ISSN":["0925-2312"],"issn-type":[{"value":"0925-2312","type":"print"}],"subject":[],"published":{"date-parts":[[2024,2]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"RoFormer: Enhanced transformer with Rotary Position Embedding","name":"articletitle","label":"Article Title"},{"value":"Neurocomputing","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.neucom.2023.127063","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2023 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}],"article-number":"127063"}}