{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T00:54:27Z","timestamp":1773708867361,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["No. 62102157"],"award-info":[{"award-number":["No. 62102157"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510050","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"2377-2388","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":64,"title":["What do they capture?"],"prefix":"10.1145","author":[{"given":"Yao","family":"Wan","sequence":"first","affiliation":[{"name":"Huazhong University of Science and Technology, China"}]},{"given":"Wei","family":"Zhao","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, China"}]},{"given":"Hongyu","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Newcastle, Australia"}]},{"given":"Yulei","family":"Sui","sequence":"additional","affiliation":[{"name":"University of Technology Sydney, Australia"}]},{"given":"Guandong","family":"Xu","sequence":"additional","affiliation":[{"name":"University of Technology Sydney, Australia"}]},{"given":"Hai","family":"Jin","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, China"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of International Conference on Learning Representations.","author":"Alon Uri","year":"2018","unstructured":"Uri Alon, Shaked Brody, Omer Levy, and Eran Yahav. 2018. code2seq: Generating Sequences from Structured Representations of Code. In Proceedings of International Conference on Learning Representations."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468611"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP. 276--286","author":"Clark Kevin","unstructured":"Kevin Clark, Urvashi Khandelwal, Omer Levy, and Christopher D. Manning. 2019. What Does BERT Look at? An Analysis of BERT's Attention. In Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP. 276--286."},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of International Conference on Learning Representations.","author":"Clark Kevin","unstructured":"Kevin Clark, Minh-Thang Luong, Quoc V. Le, and Christopher D. Manning. 2020. ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators. In Proceedings of International Conference on Learning Representations."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1198"},{"key":"e_1_3_2_1_7_1","volume-title":"Foreman","author":"Corder Gregory W.","year":"2014","unstructured":"Gregory W. Corder and Dale I. Foreman. 2014. Nonparametric statistics: A step-by-step approach. John Wiley & Sons."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4171--4186","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4171--4186."},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of Advances in Neural Information Processing Systems. 13042--13054","author":"Dong Li","year":"2019","unstructured":"Li Dong, Nan Yang, Wenhui Wang, Furu Wei, Xiaodong Liu, Yu Wang, Jianfeng Gao, Ming Zhou, and Hsiao-Wuen Hon. 2019. Unified Language Model Pretraining for Natural Language Understanding and Generation. In Proceedings of Advances in Neural Information Processing Systems. 13042--13054."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1006"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180167"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of 9th International Conference on Learning Representations.","author":"Guo Daya","year":"2021","unstructured":"Daya Guo, Shuo Ren, Shuai Lu, Zhangyin Feng, Duyu Tang, Shujie Liu, Long Zhou, Nan Duan, Alexey Svyatkovskiy, Shengyu Fu, Michele Tufano, Shao Kun Deng, Colin B. Clement, Dawn Drain, Neel Sundaresan, Jian Yin, Daxin Jiang, and Ming Zhou. 2021. GraphCodeBERT: Pre-training Code Representations with Data Flow. In Proceedings of 9th International Conference on Learning Representations."},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4129--4138","author":"Hewitt John","unstructured":"John Hewitt and Christopher D. Manning. 2019. A Structural Probe for Finding Syntax in Word Representations. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 4129--4138."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-demos.22"},{"key":"e_1_3_2_1_16_1","volume-title":"Bowman","author":"Htut Phu Mon","year":"2019","unstructured":"Phu Mon Htut, Jason Phang, Shikha Bordia, and Samuel R. Bowman. 2019. Do Attention Heads in BERT Track Syntactic Dependencies? CoRR abs\/1911.12246 (2019)."},{"key":"e_1_3_2_1_17_1","volume-title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search. CoRR abs\/1909.09436","author":"Husain Hamel","year":"2019","unstructured":"Hamel Husain, Ho-Hsiang Wu, Tiferet Gazit, Miltiadis Allamanis, and Marc Brockschmidt. 2019. CodeSearchNet Challenge: Evaluating the State of Semantic Code Search. CoRR abs\/1909.09436 (2019)."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning","volume":"119","author":"Kanade Aditya","year":"2020","unstructured":"Aditya Kanade, Petros Maniatis, Gogul Balakrishnan, and Kensen Shi. 2020. Learning and Evaluating Contextual Embedding of Source Code. In Proceedings of the 37th International Conference on Machine Learning, Vol. 119. 5110--5121."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00011"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of 8th International Conference on Learning Representations.","author":"Kim Taeuk","year":"2020","unstructured":"Taeuk Kim, Jihun Choi, Daniel Edmiston, and Sang-goo Lee. 2020. Are Pre-trained Language Models Aware of Phrases? Simple but Strong Baselines for Grammar Induction. In Proceedings of 8th International Conference on Learning Representations."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1445"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-1166-2"},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT. 1073--1094","author":"Liu Nelson F.","unstructured":"Nelson F. Liu, Matt Gardner, Yonatan Belinkov, Matthew E. Peters, and Noah A. Smith. 2019. Linguistic Knowledge and Transferability of Contextual Representations. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT. 1073--1094."},{"key":"e_1_3_2_1_24_1","volume-title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach. CoRR abs\/1907.11692","author":"Liu Yinhan","year":"2019","unstructured":"Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. CoRR abs\/1907.11692 (2019)."},{"key":"e_1_3_2_1_25_1","volume-title":"Foundations of statistical natural language processing","author":"Manning Christopher","unstructured":"Christopher Manning and Hinrich Schutze. 1999. Foundations of statistical natural language processing. MIT press."},{"key":"e_1_3_2_1_26_1","volume-title":"BERT? Assessing BERT as a Distributional Semantics Model. CoRR abs\/1911.05758","author":"Mickus Timothee","year":"2019","unstructured":"Timothee Mickus, Denis Paperno, Mathieu Constant, and Kees van Deemter. 2019. What do you mean, BERT? Assessing BERT as a Distributional Semantics Model. CoRR abs\/1911.05758 (2019)."},{"key":"e_1_3_2_1_27_1","volume-title":"Efficient Estimation of Word Representations in Vector Space. In 1st International Conference on Learning Representations, Workshop Track Proceedings.","author":"Mikolov Tom\u00e1s","year":"2013","unstructured":"Tom\u00e1s Mikolov, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. Efficient Estimation of Word Representations in Vector Space. In 1st International Conference on Learning Representations, Workshop Track Proceedings."},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of Advances in Neural Information Processing Systems. 3111--3119","author":"Mikolov Tom\u00e1s","year":"2013","unstructured":"Tom\u00e1s Mikolov, Ilya Sutskever, Kai Chen, Gregory S. Corrado, and Jeffrey Dean. 2013. Distributed Representations of Words and Phrases and their Compositionality. In Proceedings of Advances in Neural Information Processing Systems. 3111--3119."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1202"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594321"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00349"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1162"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1108"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning","volume":"97","author":"Song Kaitao","year":"2019","unstructured":"Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, and Tie-Yan Liu. 2019. MASS: Masked Sequence to Sequence Pre-training for Language Generation. In Proceedings of the 36th International Conference on Machine Learning, Vol. 97. PMLR, 5926--5936."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428301"},{"key":"e_1_3_2_1_36_1","volume-title":"ERNIE: Enhanced Representation through Knowledge Integration. CoRR abs\/1904.09223","author":"Sun Yu","year":"2019","unstructured":"Yu Sun, Shuohuan Wang, Yu-Kun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, and Hua Wu. 2019. ERNIE: Enhanced Representation through Knowledge Integration. CoRR abs\/1904.09223 (2019)."},{"key":"e_1_3_2_1_37_1","volume-title":"Le","author":"Sutskever Ilya","year":"2014","unstructured":"Ilya Sutskever, Oriol Vinyals, and Quoc V. Le. 2014. Sequence to sequence learning with neural networks. In Proceedings of Advances in neural information processing systems. 3104--3112."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1452"},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of Advances in neural information processing systems. 5998--6008","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Proceedings of Advances in neural information processing systems. 5998--6008."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-4808"},{"key":"e_1_3_2_1_41_1","volume-title":"Proceedings of 9th International Conference on Learning Representations.","author":"Vig Jesse","year":"2021","unstructured":"Jesse Vig, Ali Madani, Lav R. Varshney, Caiming Xiong, Richard Socher, and Nazneen Fatema Rajani. 2021. BERTology Meets Biology: Interpreting Attention in Protein Language Models. In Proceedings of 9th International Conference on Learning Representations."},{"key":"e_1_3_2_1_42_1","volume-title":"Proceedings of 44th International Conference on Software Engineering, Companion Volume. ACM.","author":"Wan Yao","unstructured":"Yao Wan, Yang He, Zhangqian Bi, Jianguo Zhang, Yulei Sui, Hongyu Zhang, Kazuma Hashimoto, Hai Jin, Guandong Xu, Caiming Xiong, and Philip S. Yu. 2022. NaturalCC: An Open-Source Toolkit for Code Intelligence. In Proceedings of 44th International Conference on Software Engineering, Companion Volume. ACM."},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of 34th IEEE\/ACM International Conference on Automated Software Engineering. IEEE, 13--25","author":"Wan Yao","unstructured":"Yao Wan, Jingdong Shu, Yulei Sui, Guandong Xu, Zhou Zhao, Jian Wu, and Philip S. Yu. 2019. Multi-modal Attention Network Learning for Semantic Source Code Retrieval. In Proceedings of 34th IEEE\/ACM International Conference on Automated Software Engineering. IEEE, 13--25."},{"key":"e_1_3_2_1_44_1","volume-title":"Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering. ACM, 397--407","author":"Wan Yao","unstructured":"Yao Wan, Zhou Zhao, Min Yang, Guandong Xu, Haochao Ying, Jian Wu, and Philip S. Yu. 2018. Improving automatic source code summarization via deep reinforcement learning. In Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering. ACM, 397--407."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380383"}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510050","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510050","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:04Z","timestamp":1750191124000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510050"}},"subtitle":["a structural analysis of pre-trained language models for source code"],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":45,"alternative-id":["10.1145\/3510003.3510050","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510050","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}