{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,2]],"date-time":"2026-11-02T08:28:20Z","timestamp":1793608100224,"version":"4.5.0"},"reference-count":58,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2027,2,1]],"date-time":"2027-02-01T00:00:00Z","timestamp":1801440000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100013088","name":"Qinglan Project of Jiangsu Province of China","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100013088","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["No.62272201"],"award-info":[{"award-number":["No.62272201"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Neural Networks"],"published-print":{"date-parts":[[2027,2]]},"DOI":"10.1016\/j.neunet.2026.109682","type":"journal-article","created":{"date-parts":[[2026,9,24]],"date-time":"2026-09-24T06:36:58Z","timestamp":1790231818000},"page":"109682","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"PA","title":["The role of grammar in transformer encoders: Does it really matter for language understanding process?"],"prefix":"10.1016","volume":"206","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9937-8534","authenticated-orcid":false,"given":"Fanyu","family":"Wang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chetan","family":"Arora","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9481-9599","authenticated-orcid":false,"given":"Zhenping","family":"Xie","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.neunet.2026.109682_bib0001","unstructured":"Achiam, J., Adler, S., Agarwal, S., Ahmad, L., Akkaya, I., Aleman, F. L., Almeida, D., Altenschmidt, J., Altman, S., Anadkat, S. et al. (2023). Gpt-4 technical report. arXiv: 2303.08774."},{"key":"10.1016\/j.neunet.2026.109682_bib0002","series-title":"Dependency and valency: an international handbook of contemporary research","volume":"vol. 1","author":"\u00c1gel","year":"2006"},{"key":"10.1016\/j.neunet.2026.109682_bib0003","series-title":"Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition (CVPR)","first-page":"782","article-title":"Transformer interpretability beyond attention visualization","author":"Chefer","year":"2021"},{"key":"10.1016\/j.neunet.2026.109682_bib0004","doi-asserted-by":"crossref","DOI":"10.7717\/peerj-cs.968","article-title":"Building a korean morphological analyzer using two korean BERT models","volume":"8","author":"Choi","year":"2022","journal-title":"PeerJ Computer Science"},{"key":"10.1016\/j.neunet.2026.109682_bib0005","series-title":"Proceedings of the 2019 ACL workshop blackboxNLP: analyzing and interpreting neural networks for NLP","first-page":"276","article-title":"What does BERT look at? an analysis of BERT\u2019s attention","author":"Clark","year":"2019"},{"key":"10.1016\/j.neunet.2026.109682_bib0006","doi-asserted-by":"crossref","unstructured":"Clark, K., Khandelwal, U., Levy, O., & Manning, C. D. (2019b). What does bert look at? an analysis of bert\u2019s attention. arXiv: 1906.04341.","DOI":"10.18653\/v1\/W19-4828"},{"key":"10.1016\/j.neunet.2026.109682_bib0007","unstructured":"Clark, S. (2021). Something old, something new: Grammar-based CCG parsing with transformer models. arXiv: 2109.10044."},{"issue":"1","key":"10.1016\/j.neunet.2026.109682_bib0008","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1093\/llc\/fqp031","article-title":"Treeform: Explaining and exploring grammar through syntax trees","volume":"25","author":"Derrick","year":"2010","journal-title":"Literary and Linguistic Computing"},{"key":"10.1016\/j.neunet.2026.109682_bib0009","series-title":"Proceedings of the 2019 conference of the north American chapter of the association for computational linguistics: Human language technologies, volume 1 (long and short papers)","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"issue":"3","key":"10.1016\/j.neunet.2026.109682_bib0010","doi-asserted-by":"crossref","first-page":"733","DOI":"10.1162\/coli_a_00445","article-title":"Position information in transformers: An overview","volume":"48","author":"Dufter","year":"2022","journal-title":"Computational Linguistics"},{"issue":"10","key":"10.1016\/j.neunet.2026.109682_bib0011","doi-asserted-by":"crossref","first-page":"635","DOI":"10.1002\/lnco.362","article-title":"Vector space models of word meaning and phrase meaning: A survey","volume":"6","author":"Erk","year":"2012","journal-title":"Language and Linguistics Compass"},{"issue":"7","key":"10.1016\/j.neunet.2026.109682_bib0012","doi-asserted-by":"crossref","first-page":"1079","DOI":"10.1177\/0956797612463705","article-title":"A noisy-channel account of crosslinguistic word-order variation","volume":"24","author":"Gibson","year":"2013","journal-title":"Psychological Science"},{"key":"10.1016\/j.neunet.2026.109682_bib0013","series-title":"International conference on artificial general intelligence","first-page":"153","article-title":"Guiding symbolic natural language grammar induction via transformer-based sequence probabilities","author":"Goertzel","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0014","unstructured":"Grattafiori, A., Dubey, A., Jauhri, A., Pandey, A., Kadian, A., Al-Dahle, A., Letman, A., Mathur, A., Schelten, A., Vaughan, A. et al. (2024). The llama 3 herd of models. arXiv: 2407.21783."},{"key":"10.1016\/j.neunet.2026.109682_bib0015","series-title":"Proceedings of the 58th annual meeting of the association for computational linguistics","first-page":"8342","article-title":"Don\u2019t stop pretraining: Adapt language models to domains and tasks","author":"Gururangan","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0016","unstructured":"Harer, J., Reale, C., & Chin, P. (2019). Tree-transformer: A transformer-based method for correction of tree-structured data. arXiv: 1908.00449."},{"key":"10.1016\/j.neunet.2026.109682_bib0017","unstructured":"He, P., Gao, J., & Chen, W. (2021). DeBERTaV3: Improving DeBERTa using ELECTRA-style pre-training with gradient-disentangled embedding sharing. arXiv: 2111.09543."},{"key":"10.1016\/j.neunet.2026.109682_bib0018","series-title":"Proceedings of the 2019 conference of the north american chapter of the association for computational linguistics: human language technologies, volume 1 (long and short papers)","first-page":"4129","article-title":"A structural probe for finding syntax in word representations","author":"Hewitt","year":"2019"},{"issue":"2","key":"10.1016\/j.neunet.2026.109682_bib0019","doi-asserted-by":"crossref","first-page":"360","DOI":"10.1017\/S1351324922000080","article-title":"Enhancing deep neural networks with morphological information","volume":"29","author":"Klemen","year":"2023","journal-title":"Natural Language Engineering"},{"key":"10.1016\/j.neunet.2026.109682_bib0020","series-title":"Proceedings of the 2020 conference on empirical methods in natural language processing (EMNLP)","first-page":"7057","article-title":"Attention is not only a weight: Analyzing transformers with vector norms","author":"Kobayashi","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0021","series-title":"International conference on learning representations","article-title":"Albert: A lite bert for self-supervised learning of language representations","author":"Lan","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0022","doi-asserted-by":"crossref","unstructured":"Li, B., Kim, T., Amplayo, R. K., & Keller, F. (2020). Heads-up! unsupervised constituency parsing via self-attention heads. arXiv: 2010.09517.","DOI":"10.18653\/v1\/2020.aacl-main.43"},{"key":"10.1016\/j.neunet.2026.109682_bib0023","series-title":"Findings of the association for computational linguistics: EMNLP 2020","first-page":"2710","article-title":"Universal dependencies according to BERT: Both more specific and more general","author":"Limisiewicz","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0024","doi-asserted-by":"crossref","first-page":"521","DOI":"10.1162\/tacl_a_00115","article-title":"Assessing the ability of LSTMs to learn syntax-sensitive dependencies","volume":"4","author":"Linzen","year":"2016","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.neunet.2026.109682_bib0025","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., & Stoyanov, V. (2019). Roberta: A robustly optimized bert pretraining approach. arXiv: 1907.11692."},{"key":"10.1016\/j.neunet.2026.109682_bib0026","unstructured":"Luo, X., Ramscar, M., & Love, B. C. (2024). Beyond human-like processing: Large language models perform equivalently on forward and backward scientific text. arXiv: 2411.11061."},{"key":"10.1016\/j.neunet.2026.109682_bib0027","series-title":"Proceedings of the 49th annual meeting of the association for computational linguistics: Human language technologies","first-page":"142","article-title":"Learning word vectors for sentiment analysis","author":"Maas","year":"2011"},{"key":"10.1016\/j.neunet.2026.109682_bib0028","series-title":"Proceedings of 52nd annual meeting of the association for computational linguistics: System demonstrations","first-page":"55","article-title":"The stanford coreNLP natural language processing toolkit","author":"Manning","year":"2014"},{"key":"10.1016\/j.neunet.2026.109682_bib0029","series-title":"Proceedings of the 2018 EMNLP workshop blackboxNLP: Analyzing and interpreting neural networks for NLP","first-page":"347","article-title":"Extracting syntactic trees from transformer encoder self-attentions","author":"Mare\u010dek","year":"2018"},{"issue":"8","key":"10.1016\/j.neunet.2026.109682_bib0030","doi-asserted-by":"crossref","first-page":"1388","DOI":"10.1111\/j.1551-6709.2010.01106.x","article-title":"Composition in distributional models of semantics","volume":"34","author":"Mitchell","year":"2010","journal-title":"Cognitive Science"},{"key":"10.1016\/j.neunet.2026.109682_bib0031","doi-asserted-by":"crossref","unstructured":"Mosbach, M., Degaetano-Ortlieb, S., Krielke, M.-P., Abdullah, B. M., & Klakow, D. (2020). A closer look at linguistic knowledge in masked language models: The case of relative clauses in american english. arXiv: 2011.00960.","DOI":"10.18653\/v1\/2020.coling-main.67"},{"key":"10.1016\/j.neunet.2026.109682_bib0032","unstructured":"Murty, S., Sharma, P., Andreas, J., & Manning, C. D. (2022). Characterizing intrinsic compositionality in transformers with tree projections. arXiv: 2211.01288."},{"issue":"2","key":"10.1016\/j.neunet.2026.109682_bib0033","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1080\/02671522.2011.637640","article-title":"Re-thinking grammar: The impact of embedded grammar teaching on students\u2019 writing and students\u2019 metalinguistic understanding","volume":"27","author":"Myhill","year":"2012","journal-title":"Research Papers in Education"},{"key":"10.1016\/j.neunet.2026.109682_bib0034","series-title":"Proceedings of the 60th annual meeting of the association for computational linguistics (volume 2: Short papers)","first-page":"636","article-title":"When classifying grammatical role, BERT doesn\u2019t care about word order...except when it matters","author":"Papadimitriou","year":"2022"},{"key":"10.1016\/j.neunet.2026.109682_bib0035","series-title":"Understanding English grammar: A linguistic introduction","author":"Payne","year":"2011"},{"key":"10.1016\/j.neunet.2026.109682_bib0036","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I. et al. (2018). Improving language understanding by generative pre-training."},{"issue":"5","key":"10.1016\/j.neunet.2026.109682_bib0037","doi-asserted-by":"crossref","first-page":"206","DOI":"10.1038\/s42256-019-0048-x","article-title":"Stop explaining black box machine learning models for high stakes decisions and use interpretable models instead","volume":"1","author":"Rudin","year":"2019","journal-title":"Nature Machine Intelligence"},{"key":"10.1016\/j.neunet.2026.109682_bib0038","doi-asserted-by":"crossref","first-page":"1423","DOI":"10.1162\/tacl_a_00526","article-title":"Transformer grammars: Augmenting transformer language models with syntactic inductive biases at scale","volume":"10","author":"Sartran","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.neunet.2026.109682_bib0039","unstructured":"Satharasi, T., & Iyengar, S. S. (2025). Future of AI models: A computational perspective on model collapse. arXiv: 2511.05535."},{"key":"10.1016\/j.neunet.2026.109682_bib0040","article-title":"Grammar matters","author":"Savage","year":"2010","journal-title":"New York: Cambridge UniversityPress"},{"key":"10.1016\/j.neunet.2026.109682_bib0041","doi-asserted-by":"crossref","DOI":"10.1016\/j.jml.2020.104171","article-title":"How emotion is learned: Semantic learning of novel words in emotional contexts","volume":"115","author":"Snefjella","year":"2020","journal-title":"Journal of Memory and Language"},{"key":"10.1016\/j.neunet.2026.109682_bib0042","unstructured":"Sun, Y., Wang, S., Li, Y., Feng, S., Chen, X., Zhang, H., Tian, X., Zhu, D., Tian, H., & Wu, H. (2019). Ernie: Enhanced representation through knowledge integration. arXiv: 1904.09223."},{"key":"10.1016\/j.neunet.2026.109682_bib0043","unstructured":"Sundararaman, D., Subramanian, V., Wang, G., Si, S., Shen, D., Wang, D., & Carin, L. (2019). Syntax-infused transformer and bert models for machine translation and natural language understanding. arXiv: 1911.06156."},{"key":"10.1016\/j.neunet.2026.109682_bib0044","doi-asserted-by":"crossref","DOI":"10.1016\/j.cognition.2022.105055","article-title":"Redundancy can benefit learning: Evidence from word order and case marking","volume":"224","author":"Tal","year":"2022","journal-title":"Cognition"},{"key":"10.1016\/j.neunet.2026.109682_bib0045","doi-asserted-by":"crossref","unstructured":"Tenney, I., Das, D., & Pavlick, E. (2019a). Bert rediscovers the classical nlp pipeline. arXiv: 1905.05950.","DOI":"10.18653\/v1\/P19-1452"},{"key":"10.1016\/j.neunet.2026.109682_bib0046","unstructured":"Tenney, I., Xia, P., Chen, B., Wang, A., Poliak, A., McCoy, R. T., Kim, N., Van Durme, B., Bowman, S. R., Das, D. et al. (2019b). What do you learn from context? probing for sentence structure in contextualized word representations. arXiv: 1905.06316."},{"key":"10.1016\/j.neunet.2026.109682_bib0047","article-title":"Attention is all you need","volume":"30","author":"Vaswani","year":"2017","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.neunet.2026.109682_bib0048","doi-asserted-by":"crossref","unstructured":"Vig, J., & Belinkov, Y. (2019). Analyzing the structure of attention in a transformer language model. arXiv: 1906.04284.","DOI":"10.18653\/v1\/W19-4808"},{"key":"10.1016\/j.neunet.2026.109682_bib0049","series-title":"GLUE: A multi-task benchmark and analysis platform for natural language understanding","author":"Wang","year":"2019"},{"key":"10.1016\/j.neunet.2026.109682_bib0050","series-title":"Proceedings of the 2019 conference on empirical methods in natural language processing and the 9th international joint conference on natural language processing (EMNLP-IJCNLP)","first-page":"1061","article-title":"Tree transformer: Integrating tree structures into self-attention","author":"Wang","year":"2019"},{"key":"10.1016\/j.neunet.2026.109682_bib0051","unstructured":"Warner, B., Chaffin, A., Clavie, B., Weller, O., Hallstrom, O., Taghadouini, S., Gallagher, A., Biswas, R., Ladhak, F., Aarsen, T., Cooper, N., Adams, G., Howard, J., & Poli, I. (2024). Smarter, better, faster, longer: A modern bidirectional encoder for fast, memory efficient, and long context finetuning and inference. arXiv: 2412.13663."},{"key":"10.1016\/j.neunet.2026.109682_bib0052","unstructured":"Warstadt, A., Singh, A., & Bowman, S. R. (2018). Neural network acceptability judgments. arXiv: 1805.12471."},{"key":"10.1016\/j.neunet.2026.109682_bib0053","series-title":"Proceedings of the 2020 conference on empirical methods in natural language processing: System demonstrations","first-page":"38","article-title":"Transformers: State-of-the-art natural language processing","author":"Wolf","year":"2020"},{"key":"10.1016\/j.neunet.2026.109682_bib0054","series-title":"Annual meeting of the association for computational linguistics (ACL)","first-page":"1341","article-title":"Sequence-based structured prediction for semantic parsing","author":"Xiao","year":"2016"},{"key":"10.1016\/j.neunet.2026.109682_bib0055","series-title":"Proceedings of the 2022 conference on empirical methods in natural language processing","first-page":"7576","article-title":"Parameter-efficient tuning makes a good classification head","author":"Yang","year":"2022"},{"key":"10.1016\/j.neunet.2026.109682_bib0056","series-title":"Proceedings of the 2024 conference on empirical methods in natural language processing: Industry track","first-page":"1393","article-title":"MGTE: Generalized long-context text representation and reranking models for multilingual text retrieval","author":"Zhang","year":"2024"},{"key":"10.1016\/j.neunet.2026.109682_bib0057","article-title":"Character-level convolutional networks for text classification","volume":"28","author":"Zhang","year":"2015","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.neunet.2026.109682_bib0058","doi-asserted-by":"crossref","first-page":"641","DOI":"10.1162\/tacl_a_00389","article-title":"Self-supervised regularization for text classification","volume":"9","author":"Zhou","year":"2021","journal-title":"Transactions of the Association for Computational Linguistics"}],"container-title":["Neural Networks"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608026011378?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608026011378?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,11,2]],"date-time":"2026-11-02T07:53:29Z","timestamp":1793606009000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0893608026011378"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2027,2]]},"references-count":58,"alternative-id":["S0893608026011378"],"URL":"https:\/\/doi.org\/10.1016\/j.neunet.2026.109682","relation":{},"ISSN":["0893-6080"],"issn-type":[{"value":"0893-6080","type":"print"}],"subject":[],"published":{"date-parts":[[2027,2]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"The role of grammar in transformer encoders: Does it really matter for language understanding process?","name":"articletitle","label":"Article Title"},{"value":"Neural Networks","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.neunet.2026.109682","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Ltd. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"109682"}}