{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,31]],"date-time":"2026-03-31T23:44:34Z","timestamp":1775000674660,"version":"3.50.1"},"reference-count":81,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100010256","name":"Guangzhou Youth Science and Technology Education Project","doi-asserted-by":"publisher","award":["KP2023243"],"award-info":[{"award-number":["KP2023243"]}],"id":[{"id":"10.13039\/501100010256","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1109\/tnnls.2024.3513912","type":"journal-article","created":{"date-parts":[[2024,12,17]],"date-time":"2024-12-17T14:19:40Z","timestamp":1734445180000},"page":"14988-15000","source":"Crossref","is-referenced-by-count":1,"title":["Weird-Net: Weighted Relative Distance Attention for Efficient and Robust Sequence Processing"],"prefix":"10.1109","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3165-8799","authenticated-orcid":false,"given":"Lingkai","family":"Hu","sequence":"first","affiliation":[{"name":"School of Mechanical and Electrical Engineering, Guangzhou University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0374-1348","authenticated-orcid":false,"given":"Feng","family":"Zhan","sequence":"additional","affiliation":[{"name":"School of Mechanical and Electrical Engineering, Guangzhou University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3111-7511","authenticated-orcid":false,"given":"Wenkai","family":"Huang","sequence":"additional","affiliation":[{"name":"School of Mechanical and Electrical Engineering, Guangzhou University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3164-7328","authenticated-orcid":false,"given":"Weiming","family":"Gan","sequence":"additional","affiliation":[{"name":"School of Mechanical and Electrical Engineering, Guangzhou University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-6957-9241","authenticated-orcid":false,"given":"Haoxiang","family":"Hu","sequence":"additional","affiliation":[{"name":"College of Information Science and Technology, Beijing University of Chemical Technology, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hao","family":"He","sequence":"additional","affiliation":[{"name":"School of Politics and International Relations, East China Normal University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7501-0009","authenticated-orcid":false,"given":"Kunbo","family":"Han","sequence":"additional","affiliation":[{"name":"School of Mechanical and Electrical Engineering, Guangzhou University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"An empirical evaluation of generic convolutional and recurrent networks for sequence modeling","author":"Bai","year":"2018","journal-title":"arXiv:1803.01271"},{"key":"ref2","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv:1810.04805"},{"key":"ref3","volume-title":"Improving language understanding by generative pre-training","author":"Radford","year":"2018"},{"issue":"8","key":"ref4","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2990746"},{"key":"ref6","article-title":"Empirical evaluation of gated recurrent neural networks on sequence modeling","author":"Chung","year":"2014","journal-title":"arXiv:1412.3555"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICAICA.2019.8873454"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2958103"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2024.109489"},{"key":"ref10","first-page":"3104","article-title":"Sequence to sequence learning with neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Sutskever"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1166"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2957276"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2019.2908700"},{"key":"ref15","article-title":"Conditional generative adversarial nets","author":"Mirza","year":"2014","journal-title":"arXiv:1411.1784"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.metrad.2023.100017"},{"key":"ref17","article-title":"Recurrent neural networks (RNNs): A gentle introduction and overview","author":"Schmidt","year":"2019","journal-title":"arXiv:1912.05911"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3530811"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00306"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2022.10.001"},{"key":"ref22","first-page":"16344","article-title":"FlashAttention: Fast and memory-efficient exact attention with IO-awareness","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Dao"},{"key":"ref23","article-title":"Fastformer: Additive attention can be all you need","author":"Wu","year":"2021","journal-title":"arXiv:2108.09084"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01057"},{"key":"ref25","article-title":"Generating long sequences with sparse transformers","author":"Child","year":"2019","journal-title":"arXiv:1904.10509"},{"key":"ref26","article-title":"Longformer: The long-document transformer","author":"Beltagy","year":"2020","journal-title":"arXiv:2004.05150"},{"key":"ref27","article-title":"Linformer: Self-attention with linear complexity","author":"Wang","year":"2020","journal-title":"arXiv:2006.04768"},{"key":"ref28","article-title":"Reformer: The efficient transformer","author":"Kitaev","year":"2020","journal-title":"arXiv:2001.04451"},{"key":"ref29","first-page":"9438","article-title":"Sparse Sinkhorn attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Tay"},{"key":"ref30","article-title":"Synthesizer: Rethinking self-attention in transformer models","author":"Tay","year":"2020","journal-title":"arXiv:2005.00743"},{"key":"ref31","first-page":"17283","article-title":"Big Bird: Transformers for longer sequences","volume-title":"Proc. 34th Conf. Neural Inf. Process. Syst. (NeurlPS)","author":"Zaheer"},{"key":"ref32","first-page":"5156","article-title":"Transformers are RNNs: Fast autoregressive transformers with linear attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Katharopoulos"},{"key":"ref33","article-title":"Masked language modeling for proteins via linearly scalable long-context transformers","author":"Choromanski","year":"2020","journal-title":"arXiv:2006.03555"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.trc.2019.08.005"},{"key":"ref35","first-page":"13203","article-title":"Staircase attention for recurrent processing of sequences","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Ju"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.3390\/e23030283"},{"key":"ref37","article-title":"Long range arena: A benchmark for efficient transformers","author":"Tay","year":"2020","journal-title":"arXiv:2011.04006"},{"key":"ref38","article-title":"DropKey","author":"Li","year":"2022","journal-title":"arXiv:2208.02646"},{"key":"ref39","volume-title":"Enwik8.datasets At Hugging Face","year":"2023"},{"key":"ref40","volume-title":"Zhwiki520","author":"Hu","year":"2020"},{"key":"ref41","article-title":"Structured attention networks","author":"Kim","year":"2017","journal-title":"arXiv:1702.00887"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2021.03.091"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.3389\/fncom.2020.00029"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2020.3007321"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.3390\/rs13152965"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.3019893"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045336"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2962815"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2991083"},{"key":"ref50","article-title":"Lessons on parameter sharing across layers in transformers","author":"Takase","year":"2021","journal-title":"arXiv:2104.06022"},{"key":"ref51","first-page":"10890","article-title":"R-Drop: Regularized dropout for neural networks","volume-title":"Proc. Neural Inf. Process. Syst.","volume":"34","author":"Wu"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i16.17705"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2890970"},{"key":"ref54","first-page":"3744","article-title":"Set transformer: A framework for attention-based permutation-invariant neural networks","volume-title":"Proc. Int. Conf. Mach. Learn. (ICML)","author":"Lee"},{"key":"ref55","first-page":"2441","article-title":"Luna: Linear unified nested attention","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ma"},{"key":"ref56","article-title":"LongNet: Scaling transformers to 1,000,000,000 tokens","author":"Ding","year":"2023","journal-title":"arXiv:2307.02486"},{"key":"ref57","article-title":"Retentive network: A successor to transformer for large language models","author":"Sun","year":"2023","journal-title":"arXiv:2307.08621"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00353"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1613\/jair.731"},{"key":"ref60","article-title":"Relational inductive biases, deep learning, and graph networks","author":"Battaglia","year":"2018","journal-title":"arXiv:1806.01261"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1098\/rspa.2021.0068"},{"key":"ref62","first-page":"5301","article-title":"On the spectral bias of neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rahaman"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3145365"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.3390\/info15090517"},{"key":"ref65","article-title":"Gaussian error linear units (GELUs)","author":"Hendrycks","year":"2016","journal-title":"arXiv:1606.08415"},{"key":"ref66","article-title":"Pointer sentinel mixture models","author":"Merity","year":"2016","journal-title":"arXiv:1609.07843"},{"key":"ref67","article-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2014","journal-title":"arXiv:1412.6980"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2010.11929"},{"key":"ref69","article-title":"FNet: Mixing tokens with Fourier transforms","author":"Lee-Thorp","year":"2021","journal-title":"arXiv:2105.03824"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.19"},{"key":"ref71","article-title":"CosFormer: Rethinking softmax in attention","author":"Qin","year":"2022","journal-title":"arXiv:2202.08791"},{"key":"ref72","article-title":"Flowformer: Linearizing transformers with conservation flows","author":"Wu","year":"2022","journal-title":"arXiv:2202.06258"},{"key":"ref73","article-title":"Sliceformer: Make multi-head attention as simple as sorting in discriminative tasks","author":"Yuan","year":"2023","journal-title":"arXiv:2310.17683"},{"key":"ref74","article-title":"Primal-attention: Self-attention through asymmetric kernel SVD in primal representation","author":"Chen","year":"2023","journal-title":"arXiv:2305.19798"},{"key":"ref75","article-title":"Sparse factorization of large square matrices","author":"Khalitov","year":"2021","journal-title":"arXiv:2109.08184"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i16.17664"},{"key":"ref77","article-title":"The hedgehog & the porcupine: Expressive linear attentions with softmax mimicry","author":"Zhang","year":"2024","journal-title":"arXiv:2402.04347"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i12.29244"},{"key":"ref79","first-page":"12493","article-title":"Keeping your eye on the ball: Trajectory attention in video transformers","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Patrick"},{"key":"ref80","volume-title":"Hutter Prize for Lossless Compression of Human Knowledge","year":"2006"},{"key":"ref81","article-title":"Autonomous navigation algorithm for underwater robots based on global-local fusion under multi-objective optimization","author":"Gao","year":"2024","journal-title":"Authorea Preprints"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/5962385\/11114436\/10804812.pdf?arnumber=10804812","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,6]],"date-time":"2025-08-06T18:01:08Z","timestamp":1754503268000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10804812\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8]]},"references-count":81,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2024.3513912","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8]]}}}