{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,25]],"date-time":"2025-10-25T18:46:38Z","timestamp":1761417998700,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,14]],"date-time":"2024-04-14T00:00:00Z","timestamp":1713052800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"National Science Foundation","award":["2311469"],"award-info":[{"award-number":["2311469"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,14]]},"DOI":"10.1145\/3639476.3639768","type":"proceedings-article","created":{"date-parts":[[2024,5,24]],"date-time":"2024-05-24T15:15:01Z","timestamp":1716563701000},"page":"72-76","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Which Syntactic Capabilities Are Statistically Learned by Masked Language Models for Code?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4829-1017","authenticated-orcid":false,"given":"Alejandro","family":"Velasco","sequence":"first","affiliation":[{"name":"William & Mary, Williamsburg, Virginia, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6166-7595","authenticated-orcid":false,"given":"David N.","family":"Palacio","sequence":"additional","affiliation":[{"name":"William & Mary, Williamsburg, Virginia, United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3238-1229","authenticated-orcid":false,"given":"Daniel","family":"Rodriguez-Cardenas","sequence":"additional","affiliation":[{"name":"William & Mary, Williamsburg, Virginia, United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5626-7586","authenticated-orcid":false,"given":"Denys","family":"Poshyvanyk","sequence":"additional","affiliation":[{"name":"William & Mary, Williamsburg, Virginia, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,5,24]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. WM-SEMERU\/SyntaxEval. https:\/\/github.com\/WM-SEMERU\/SyntaxEval original-date: 2022-09-09T20:53:59Z."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","unstructured":"Toufique Ahmed Dian Yu Chengxuan Huang Cathy Wang et al. 2023. Towards Understanding What Code Language Models Learned. arXiv:2306.11943 [cs]. 10.48550\/arXiv.2306.11943","DOI":"10.48550\/arXiv.2306.11943"},{"key":"e_1_3_2_1_3_1","volume-title":"Principles and Practice of Explainable Machine Learning. CoRR abs\/2009.11698","author":"Belle Vaishak","year":"2020","unstructured":"Vaishak Belle and Ioannis Papantonis. 2020. Principles and Practice of Explainable Machine Learning. CoRR abs\/2009.11698 (2020). arXiv:2009.11698 https:\/\/arxiv.org\/abs\/2009.11698"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1595696.1595728"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2940179"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"Matteo Ciniselli Nathan Cooper Luca Pascarella Antonio Mastropaolo et al. [n. d.]. An Empirical Study on the Usage of Transformer Models for Code Completion. ([n. d.]) 1--1. 10.1109\/TSE.2021.3128234","DOI":"10.1109\/TSE.2021.3128234"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Matteo Ciniselli Nathan Cooper Luca Pascarella Antonio Mastropaolo et al. 2021. An Empirical Study on the Usage of Transformer Models for Code Completion. arXiv:cs.SE\/2108.01585","DOI":"10.1109\/TSE.2021.3128234"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Matteo Ciniselli Nathan Cooper Luca Pascarella Denys Poshyvanyk et al. 2021. An Empirical Study on the Usage of BERT Models for Code Completion. CoRR abs\/2103.07115 (2021). arXiv:2103.07115 https:\/\/arxiv.org\/abs\/2103.07115","DOI":"10.1109\/MSR52588.2021.00024"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.728"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","unstructured":"Jacob Devlin Ming-Wei Chang Kenton Lee and Kristina Toutanova. [n. d.]. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv:1810.04805 [cs] 10.48550\/arXiv.1810.04805","DOI":"10.48550\/arXiv.1810.04805"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Zhangyin Feng Daya Guo Duyu Tang Nan Duan et al. 2020. CodeBERT: A Pre-Trained Model for Programming and Natural Languages. arXiv:cs.CL\/2002.08155","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2009.64"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","unstructured":"Sangmok Han David R. Wallace and Robert C. Miller. [n. d.]. Code completion of multiple keywords from abbreviated input. 18 3 ([n. d.]) 363--398. 10.1007\/s10515-011-0083-2","DOI":"10.1007\/s10515-011-0083-2"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556900"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2012.6227135"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Xinyi Hou Yanjie Zhao Yue Liu Zhou Yang et al. 2023. Large Language Models for Software Engineering: A Systematic Literature Review. http:\/\/arxiv.org\/abs\/2308.10620 arXiv:2308.10620 [cs].","DOI":"10.1145\/3695988"},{"key":"e_1_3_2_1_17_1","unstructured":"Hamel Husain Ho-Hsiang Wu Tiferet Gazit Miltiadis Allamanis et al. 2019. CodeSearchNet Challenge: Evaluating the State of Semantic Code Search. arXiv:1909.09436 [cs stat] (Sept. 2019). http:\/\/arxiv.org\/abs\/1909.09436 arXiv:1909.09436."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Masahiro Kaneko Masato Mita Shun Kiyono Jun Suzuki et al. 2020. Encoder-Decoder Models Can Benefit from Pre-trained Masked Language Models in Grammatical Error Correction. arXiv:cs.CL\/2005.00987","DOI":"10.18653\/v1\/2020.acl-main.391"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3341624"},{"key":"e_1_3_2_1_20_1","unstructured":"Yinhan Liu Myle Ott Naman Goyal Jingfei Du et al. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:cs.CL\/1907.11692"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","unstructured":"Yinhan Liu Myle Ott Naman Goyal Jingfei Du et al. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs]. 10.48550\/arXiv.1907.11692","DOI":"10.48550\/arXiv.1907.11692"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","unstructured":"Wei Ma Mengjie Zhao Xiaofei Xie Qiang Hu et al. 2023. Are Code Pre-trained Models Powerful to Learn Code Syntax and Semantics? arXiv:2212.10017 [cs]. 10.48550\/arXiv.2212.10017","DOI":"10.48550\/arXiv.2212.10017"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/icse43902.2021.00041"},{"key":"e_1_3_2_1_24_1","unstructured":"Ahmad Haji Mohammadkhani and Hadi Hemmati. [n. d.]. Explainable AI for Pre-Trained Code Models: What Do They Learn? When They Do Not Work? ([n. d.])."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","unstructured":"David N. Palacio Nathan Cooper Alvaro Rodriguez Kevin Moran et al. [n. d.]. Toward a Theory of Causation for Interpreting Neural Code Models. arXiv:2302.03788 [cs stat] 10.48550\/arXiv.2302.03788","DOI":"10.48550\/arXiv.2302.03788"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Judea Pearl. 2009. Causality: models reasoning and inference.","DOI":"10.1017\/CBO9780511803161"},{"key":"e_1_3_2_1_27_1","unstructured":"Rafiqul Islam Rabin Arjun Mukherjee Omprakash Gnawali and Mohammad Amin Alipour. [n. d.]. Towards Demystifying Dimensions of Source Code Embeddings. ([n. d.]) 29--38. ISBN: 9781450381253."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594321"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME58846.2023.00040"},{"key":"e_1_3_2_1_30_1","unstructured":"Amit Sharma Vasilis Syrgkanis Cheng Zhang and Emre K\u0131c\u0131man. 2021. DoWhy : Addressing Challenges in Expressing and Validating Causal Assumptions. (2021)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1017\/UPO9788175968363"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417058"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330699"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.blackboxnlp-1.31"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196398.3196431"},{"key":"e_1_3_2_1_36_1","volume-title":"Towards Automating Code Review Activities. In 43rd International Conference on Software Engineering, ICSE'21","author":"Tufano Rosalia","year":"2021","unstructured":"Rosalia Tufano, Luca Pascarella, Michele Tufano, Denys Poshyvanyk, et al. 2021. Towards Automating Code Review Activities. In 43rd International Conference on Software Engineering, ICSE'21. https:\/\/arxiv.org\/abs\/2101.02518"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","unstructured":"Yao Wan Wei Zhao Hongyu Zhang Yulei Sui et al. [n. d.]. What Do They Capture? - A Structural Analysis of Pre-Trained Language Models for Source Code. arXiv:2202.06840 [cs] 10.48550\/arXiv.2202.06840","DOI":"10.48550\/arXiv.2202.06840"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485275"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380429"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2019.8668043"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970326"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.38"}],"event":{"name":"ICSE-NIER'24: 2024 ACM\/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"],"location":"Lisbon Portugal","acronym":"ICSE-NIER'24"},"container-title":["Proceedings of the 2024 ACM\/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639476.3639768","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3639476.3639768","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:53:38Z","timestamp":1750287218000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3639476.3639768"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,14]]},"references-count":42,"alternative-id":["10.1145\/3639476.3639768","10.1145\/3639476"],"URL":"https:\/\/doi.org\/10.1145\/3639476.3639768","relation":{},"subject":[],"published":{"date-parts":[[2024,4,14]]},"assertion":[{"value":"2024-05-24","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}