{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,5]],"date-time":"2025-11-05T02:46:31Z","timestamp":1762310791864,"version":"build-2065373602"},"reference-count":129,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T00:00:00Z","timestamp":1759190400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T00:00:00Z","timestamp":1759190400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000156","name":"Fonds de Recherche du Qu\u00e9bec - Sant\u00e9","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000156","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007631","name":"Canadian Institute for Advanced Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007631","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1007\/s10664-025-10731-0","type":"journal-article","created":{"date-parts":[[2025,9,30]],"date-time":"2025-09-30T06:47:13Z","timestamp":1759214833000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["DeepCodeProbe: Evaluating Code Representation Quality in Models Trained on Code"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4411-0810","authenticated-orcid":false,"given":"Vahid","family":"Majdinasab","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,9,30]]},"reference":[{"key":"10731_CR1","doi-asserted-by":"crossref","unstructured":"Ahmad WU, Chakraborty S, Ray B, Chang KW (2020) A transformer-based approach for source code summarization. arXiv:2005.00653","DOI":"10.18653\/v1\/2020.acl-main.449"},{"key":"10731_CR2","doi-asserted-by":"publisher","first-page":"86121","DOI":"10.1109\/ACCESS.2019.2918202","volume":"7","author":"QU Ain","year":"2019","unstructured":"Ain QU, Butt WH, Anwar MW, Azam F, Maqbool B (2019) A systematic review on code clone detection. IEEE Access 7:86121\u201386144","journal-title":"IEEE Access"},{"key":"10731_CR3","unstructured":"Allamanis M, Peng H, Sutton C (2016) A convolutional attention network for extreme summarization of source code. In: International conference on machine learning. PMLR, pp 2091\u20132100"},{"key":"10731_CR4","doi-asserted-by":"crossref","unstructured":"Barbez A, Khomh F, Gu\u00e9h\u00e9neuc YG (2019) Deep learning anti-patterns from code metrics history. In: 2019 IEEE International Conference on Software Maintenance and Evolution (ICSME). IEEE, pp 114\u2013124","DOI":"10.1109\/ICSME.2019.00021"},{"issue":"1","key":"10731_CR5","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1162\/coli_a_00422","volume":"48","author":"Y Belinkov","year":"2022","unstructured":"Belinkov Y (2022) Probing classifiers: promises, shortcomings, and advances. Comput Linguist 48(1):207\u2013219","journal-title":"Comput Linguist"},{"key":"10731_CR6","doi-asserted-by":"publisher","first-page":"166","DOI":"10.1007\/s10664-009-9118-8","volume":"15","author":"HC Benestad","year":"2010","unstructured":"Benestad HC, Anda B, Arisholm E (2010) Understanding cost drivers of software evolution: a quantitative and qualitative investigation of change effort in two evolving software systems. Empir Softw Eng 15:166\u2013203","journal-title":"Empir Softw Eng"},{"key":"10731_CR7","doi-asserted-by":"crossref","unstructured":"Bhatia S, Kohli P, Singh R (2018) Neuro-symbolic program corrector for introductory programming assignments. In: Proceedings of the 40th international conference on software engineering, pp 60\u201370","DOI":"10.1145\/3180155.3180219"},{"key":"10731_CR8","unstructured":"Budach L, Feuerpfeil M, Ihde N, Nathansen A, Noack N, Patzlaff H, Naumann F, Harmouch H (2022) The effects of data quality on machine learning performance. arXiv:2207.14529"},{"key":"10731_CR9","doi-asserted-by":"crossref","unstructured":"Bui ND, Yu Y, Jiang L (2021a) Infercode: Self-supervised learning of code representations by predicting subtrees. In: 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). IEEE, pp 1186\u20131197","DOI":"10.1109\/ICSE43902.2021.00109"},{"key":"10731_CR10","doi-asserted-by":"crossref","unstructured":"Bui ND, Yu Y, Jiang L (2021b) Treecaps: Tree-based capsule networks for source code processing. In: Proceedings of the AAAI conference on artificial intelligence, vol 35, pp 30\u201338","DOI":"10.1609\/aaai.v35i1.16074"},{"issue":"8","key":"10731_CR11","doi-asserted-by":"publisher","first-page":"832","DOI":"10.3390\/electronics8080832","volume":"8","author":"DV Carvalho","year":"2019","unstructured":"Carvalho DV, Pereira EM, Cardoso JS (2019) Machine learning interpretability: a survey on methods and metrics. Electronics 8(8):832","journal-title":"Electronics"},{"key":"10731_CR12","doi-asserted-by":"crossref","unstructured":"Chen Z, Cao Y, Liu Y, Wang H, Xie T, Liu X (2020) A comprehensive study on challenges in deploying deep learning based software. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 750\u2013762","DOI":"10.1145\/3368089.3409759"},{"key":"10731_CR13","doi-asserted-by":"crossref","unstructured":"Chilowicz M, Duris E, Roussel G (2009) Syntax tree fingerprinting for source code similarity detection. In: 2009 IEEE 17th international conference on program comprehension. IEEE, pp 243\u2013247","DOI":"10.1109\/ICPC.2009.5090050"},{"key":"10731_CR14","unstructured":"Choudhary S, Chatterjee N, Saha SK (2022) Interpretation of black box nlp models: a survey. arXiv:2203.17081"},{"key":"10731_CR15","doi-asserted-by":"crossref","unstructured":"Dam HK, Pham T, Ng SW, Tran T, Grundy J, Ghose A, Kim T, Kim CJ (2018) A deep tree-based model for software defect prediction. arXiv:1802.00921","DOI":"10.1109\/MSR.2019.00017"},{"key":"10731_CR16","doi-asserted-by":"crossref","unstructured":"d\u2019Eon G, d\u2019Eon J, Wright JR, Leyton-Brown K (2022) The spotlight: A general method for discovering systematic errors in deep learning models. In: Proceedings of the 2022 ACM conference on fairness, accountability, and transparency, pp 1962\u20131981","DOI":"10.1145\/3531146.3533240"},{"key":"10731_CR17","unstructured":"Devlin J, Chang MW, Lee K, Toutanova K (2018) Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv:1810.04805"},{"key":"10731_CR18","doi-asserted-by":"crossref","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D et\u00a0al (2020) Codebert: A pre-trained model for programming and natural languages. arXiv:2002.08155","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"10731_CR19","doi-asserted-by":"crossref","unstructured":"Gao Y, Wang Z, Liu S, Yang L, Sang W, Cai Y (2019) Teccd: A tree embedding approach for code clone detection. In: 2019 IEEE International Conference on Software Maintenance and Evolution (ICSME). IEEE, pp 145\u2013156","DOI":"10.1109\/ICSME.2019.00025"},{"key":"10731_CR20","doi-asserted-by":"crossref","unstructured":"Goodwin E, Sinha K, O\u2019Donnell TJ (2020) Probing linguistic systematicity. arXiv:2005.04315","DOI":"10.18653\/v1\/2020.acl-main.177"},{"key":"10731_CR21","doi-asserted-by":"crossref","unstructured":"Gros D, Sezhiyan H, Devanbu P, Yu Z (2020) Code to comment\u201c translation\u201d data, metrics, baselining & evaluation. In: Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering, pp 746\u2013757","DOI":"10.1145\/3324884.3416546"},{"key":"10731_CR22","unstructured":"Guo D, Ren S, Lu S, Feng Z, Tang D, Liu S, Zhou L, Duan N, Svyatkovskiy A Fu S, et\u00a0al. (2020) Graphcodebert: Pre-training code representations with data flow. arXiv:2009.08366"},{"key":"10731_CR23","doi-asserted-by":"crossref","unstructured":"Guo D, Lu S, Duan N, Wang Y, Zhou M, Yin J (2022) Unixcoder: Unified cross-modal pre-training for code representation. arXiv:2203.03850","DOI":"10.18653\/v1\/2022.acl-long.499"},{"key":"10731_CR24","doi-asserted-by":"crossref","unstructured":"Gupta R, Pal S, Kanade A, Shevade S (2017) Deepfix: Fixing common c language errors by deep learning. In: Proceedings of the aaai conference on artificial intelligence, vol\u00a031","DOI":"10.1609\/aaai.v31i1.10742"},{"key":"10731_CR25","unstructured":"Hall P, Gill N, Schmidt N (2019) Proposed guidelines for the responsible use of explainable machine learning. arXiv:1906.03533"},{"key":"10731_CR26","doi-asserted-by":"crossref","unstructured":"Hellendoorn VJ, Bird C, Barr ET, Allamanis M (2018) Deep learning type inference. In: Proceedings of the 2018 26th acm joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 152\u2013162","DOI":"10.1145\/3236024.3236051"},{"key":"10731_CR27","unstructured":"Hernandez D, Brown TB (2020) Measuring the algorithmic efficiency of neural networks. arXiv:2005.04305"},{"key":"10731_CR28","doi-asserted-by":"crossref","unstructured":"Hern\u00e1ndez\u00a0L\u00f3pez JA, Weyssow M, Cuadrado JS, Sahraoui H (2022) Ast-probe: Recovering abstract syntax trees from hidden representations of pre-trained language models. In: Proceedings of the 37th IEEE\/ACM International Conference on Automated Software Engineering, pp 1\u201311","DOI":"10.1145\/3551349.3556900"},{"key":"10731_CR29","unstructured":"Hestness J, Narang S, Ardalani N, Diamos G, Jun H, Kianinejad H, Patwary MMA, Yang Y, Zhou Y (2017) Deep learning scaling is predictable, empirically. arXiv:1712.00409"},{"key":"10731_CR30","doi-asserted-by":"crossref","unstructured":"Hewitt J, Liang P (2019) Designing and interpreting probes with control tasks. arXiv:1909.03368","DOI":"10.18653\/v1\/D19-1275"},{"key":"10731_CR31","unstructured":"Hewitt J, Manning CD (2019) A structural probe for finding syntax in word representations. In: Proceedings of the 2019 conference of the north american chapter of the association for computational linguistics: human language technologies, Volume 1 (Long and Short Papers), pp 4129\u20134138"},{"key":"10731_CR32","unstructured":"Hoffmann J, Borgeaud S, Mensch A, Buchatskaya E, Cai T, Rutherford E, Casas DdL, Hendricks LA, Welbl J, Clark A et\u00a0al. (2022) Training compute-optimal large language models. arXiv:2203.15556"},{"key":"10731_CR33","unstructured":"Holm S (1979) A simple sequentially rejective multiple test procedure. Scand J Stat 65\u201370"},{"key":"10731_CR34","doi-asserted-by":"crossref","unstructured":"Hu X, Li G, Xia X, Lo D, Jin Z (2018) Deep code comment generation. In: Proceedings of the 26th conference on program comprehension, pp 200\u2013210","DOI":"10.1145\/3196321.3196334"},{"issue":"1","key":"10731_CR35","doi-asserted-by":"publisher","first-page":"304","DOI":"10.1109\/TR.2020.3001918","volume":"70","author":"W Hua","year":"2020","unstructured":"Hua W, Sui Y, Wan Y, Liu G, Xu G (2020) Fcca: Hybrid code representation for functional clone detection using attention networks. IEEE Trans Reliab 70(1):304\u2013318","journal-title":"IEEE Trans Reliab"},{"key":"10731_CR36","unstructured":"Husain H, Wu HH, Gazit T, Allamanis M, Brockschmidt M (2019) Codesearchnet challenge: evaluating the state of semantic code search. arXiv:1909.09436"},{"key":"10731_CR37","doi-asserted-by":"crossref","unstructured":"Jain P, Jain A, Zhang T, Abbeel P, Gonzalez JE, Stoica I (2020) Contrastive code representation learning. arXiv:2007.04973","DOI":"10.18653\/v1\/2021.emnlp-main.482"},{"issue":"1","key":"10731_CR38","doi-asserted-by":"publisher","first-page":"2","DOI":"10.3390\/technologies9010002","volume":"9","author":"A Jaiswal","year":"2020","unstructured":"Jaiswal A, Babu AR, Zadeh MZ, Banerjee D, Makedon F (2020) A survey on contrastive self-supervised learning. Technologies 9(1):2","journal-title":"Technologies"},{"key":"10731_CR39","doi-asserted-by":"crossref","unstructured":"Johnstone IM, Titterington DM (2009) Statistical challenges of high-dimensional data","DOI":"10.1098\/rsta.2009.0159"},{"key":"10731_CR40","unstructured":"Kaplan J, McCandlish S, Henighan T, Brown TB, Chess B, Child R, Gray S, Radford A, Wu J, Amodei D (2020) Scaling laws for neural language models. arXiv:2001.08361"},{"key":"10731_CR41","doi-asserted-by":"crossref","unstructured":"Karmakar A, Robbes R (2021) What do pre-trained code models know about code? In: 2021 36th IEEE\/ACM International Conference on Automated Software Engineering (ASE). IEEE, pp 1332\u20131336","DOI":"10.1109\/ASE51524.2021.9678927"},{"issue":"2","key":"10731_CR42","doi-asserted-by":"publisher","first-page":"220","DOI":"10.1109\/TSE.2023.3341624","volume":"50","author":"A Karmakar","year":"2023","unstructured":"Karmakar A, Robbes R (2023) Inspect: intrinsic and systematic probing evaluation for code transformers. IEEE Trans Software Eng 50(2):220\u2013238","journal-title":"IEEE Trans Software Eng"},{"key":"10731_CR43","unstructured":"Kingma DP, Ba J (2014) Adam: A method for stochastic optimization. arXiv:1412.6980"},{"key":"10731_CR44","unstructured":"Koch S (2005) Evolution of open source software systems\u2013a large-scale investigation. In: Proceedings of the 1st International Conference on Open Source Systems, pp 148\u2013153"},{"key":"10731_CR45","doi-asserted-by":"publisher","first-page":"170","DOI":"10.1016\/j.jss.2016.01.003","volume":"121","author":"L Kumar","year":"2016","unstructured":"Kumar L, Rath SK (2016) Hybrid functional link artificial neural network approach for predicting maintainability of object-oriented software. J Syst Softw 121:170\u2013190","journal-title":"J Syst Softw"},{"key":"10731_CR46","unstructured":"Le Q, Mikolov T (2014) Distributed representations of sentences and documents. In: International conference on machine learning. PMLR, pp 1188\u20131196"},{"key":"10731_CR47","doi-asserted-by":"crossref","unstructured":"LeClair A, Haque S, Wu L, McMillan C (2020) Improved code summarization via a graph neural network. In: Proceedings of the 28th international conference on program comprehension, pp 184\u2013195","DOI":"10.1145\/3387904.3389268"},{"issue":"7553","key":"10731_CR48","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun Y, Bengio Y, Hinton G (2015) Deep learning. Nature 521(7553):436\u2013444","journal-title":"Nature"},{"key":"10731_CR49","doi-asserted-by":"publisher","first-page":"111141","DOI":"10.1016\/j.jss.2021.111141","volume":"184","author":"M Lei","year":"2022","unstructured":"Lei M, Li H, Li J, Aundhkar N, Kim DK (2022) Deep learning application on code clone detection: a review of current knowledge. J Syst Softw 184:111141","journal-title":"J Syst Softw"},{"key":"10731_CR50","doi-asserted-by":"crossref","unstructured":"Li L, Feng H, Zhuang W, Meng N, Ryder B (2017) Cclearner: A deep learning-based clone detection approach. In: 2017 IEEE international conference on software maintenance and evolution (ICSME). IEEE, pp 249\u2013260","DOI":"10.1109\/ICSME.2017.46"},{"key":"10731_CR51","doi-asserted-by":"crossref","unstructured":"Li Y, Wang S, Nguyen TN (2020) Dlfix: Context-based code transformation learning for automated program repair. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, pp 602\u2013614","DOI":"10.1145\/3377811.3380345"},{"key":"10731_CR52","doi-asserted-by":"crossref","unstructured":"Li X, Gong Y, Shen Y, Qiu X, Zhang H, Yao B, Qi W, Jiang D, Chen W, Duan N (2022a) Coderetriever: A large scale contrastive pre-training method for code search. In: Proceedings of the 2022 conference on empirical methods in natural language processing, pp 2898\u20132910","DOI":"10.18653\/v1\/2022.emnlp-main.187"},{"key":"10731_CR53","doi-asserted-by":"crossref","unstructured":"Li Y, Wang S, Nguyen TN (2022b) Dear: A novel deep learning-based approach for automated program repair. In: Proceedings of the 44th international conference on software engineering, pp 511\u2013523","DOI":"10.1145\/3510003.3510177"},{"issue":"1","key":"10731_CR54","doi-asserted-by":"publisher","first-page":"258","DOI":"10.1109\/TR.2022.3154773","volume":"72","author":"Z Li","year":"2022","unstructured":"Li Z, Wu Y, Peng B, Chen X, Sun Z, Liu Y, Paul D (2022) Setransformer: A transformer-based code semantic parser for code comment generation. IEEE Trans Reliab 72(1):258\u2013273","journal-title":"IEEE Trans Reliab"},{"key":"10731_CR55","unstructured":"Li R, Allal LB, Zi Y, Muennighoff N, Kocetkov D, Mou C, Marone M, Akiki C, Li J, Chim J et\u00a0al. (2023) Starcoder: may the source be with you! arXiv:2305.06161"},{"key":"10731_CR56","unstructured":"Lin J, Dong H, Xie Y, Zhang L (2024) Scaling laws behind code understanding model. arXiv:2402.12813"},{"key":"10731_CR57","doi-asserted-by":"crossref","unstructured":"Liu Y, Li Y, Guo J, Zhou Y, Xu B (2018) Connecting software metrics across versions to predict defects. In: 2018 IEEE 25th International Conference on Software Analysis. Evolution and Reengineering (SANER). IEEE, pp 232\u2013243","DOI":"10.1109\/SANER.2018.8330212"},{"key":"10731_CR58","doi-asserted-by":"crossref","unstructured":"Liu NF, Gardner M, Belinkov Y, Peters ME, Smith NA (2019) Linguistic knowledge and transferability of contextual representations. arXiv:1903.08855","DOI":"10.18653\/v1\/N19-1112"},{"key":"10731_CR59","unstructured":"Luo S, Ivison H, Han SC, Poon J (2021) Local interpretations for explainable natural language processing: a survey. ACM Comput Surv"},{"key":"10731_CR60","doi-asserted-by":"crossref","unstructured":"Lutellier T, Pham HV, Pang L, Li Y, Wei M, Tan L (2020) Coconut: combining context-aware neural translation models using ensemble for program repair. In: Proceedings of the 29th ACM SIGSOFT international symposium on software testing and analysis, pp 101\u2013114","DOI":"10.1145\/3395363.3397369"},{"issue":"7","key":"10731_CR61","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3664606","volume":"33","author":"W Ma","year":"2024","unstructured":"Ma W, Liu S, Zhao M, Xie X, Wang W, Hu Q, Zhang J, Liu Y (2024) Unveiling code pre-trained models: Investigating syntax and semantics capacities. ACM Trans Softw Eng Methodol 33(7):1\u201329","journal-title":"ACM Trans Softw Eng Methodol"},{"key":"10731_CR62","unstructured":"Majdinasab V (2024) Deepcodeprobe. https:\/\/github.com\/CommissarSilver\/DeepCodeProbe"},{"key":"10731_CR63","doi-asserted-by":"crossref","unstructured":"Mann HB, Whitney DR (1947) On a test of whether one of two random variables is stochastically larger than the other. Ann Math Stat 50\u201360","DOI":"10.1214\/aoms\/1177730491"},{"key":"10731_CR64","doi-asserted-by":"crossref","unstructured":"Maudslay RH, Valvoda J, Pimentel T, Williams A, Cotterell R (2020) A tale of a probe and a parser. arXiv:2005.01641","DOI":"10.18653\/v1\/2020.acl-main.659"},{"issue":"2","key":"10731_CR65","doi-asserted-by":"publisher","first-page":"361","DOI":"10.1037\/0033-2909.111.2.361","volume":"111","author":"KO McGraw","year":"1992","unstructured":"McGraw KO, Wong SP (1992) A common language effect size statistic. Psychol Bull 111(2):361","journal-title":"Psychol Bull"},{"key":"10731_CR66","doi-asserted-by":"crossref","unstructured":"Mens T, Wermelinger M, Ducasse S, Demeyer S, Hirschfeld R, Jazayeri M (2005) Challenges in software evolution. In: Eighth International Workshop on Principles of Software Evolution (IWPSE\u201905). IEEE, pp 13\u201322","DOI":"10.1109\/IWPSE.2005.7"},{"key":"10731_CR67","doi-asserted-by":"crossref","unstructured":"Miaschi A, Dell\u2019Orletta F (2020) Contextual and non-contextual word embeddings: an in-depth linguistic investigation. In: Proceedings of the 5th workshop on representation learning for NLP, pp 110\u2013119","DOI":"10.18653\/v1\/2020.repl4nlp-1.15"},{"key":"10731_CR68","unstructured":"Mikolov T, Sutskever I, Chen K, Corrado GS, Dean J (2013) Distributed representations of words and phrases and their compositionality. Adv Neural Inf Process Syst 26"},{"key":"10731_CR69","doi-asserted-by":"crossref","unstructured":"Mir AM, Lato\u0161kinas E, Proksch S, Gousios G (2022) Type4py: Practical deep similarity learning-based type inference for python. In: Proceedings of the 44th international conference on software engineering, pp 2241\u20132252","DOI":"10.1145\/3510003.3510124"},{"key":"10731_CR70","doi-asserted-by":"crossref","unstructured":"Mitkov R, Evans R, Or\u0103san C, Dornescu I, Rios M (2012) Coreference resolution: To what extent does it help nlp applications? In: Text, speech and dialogue: 15th International Conference, TSD 2012, Brno, Czech Republic, September 3-7, 2012. Proceedings 15. Springer, pp 16\u201327","DOI":"10.1007\/978-3-642-32790-2_2"},{"key":"10731_CR71","doi-asserted-by":"crossref","unstructured":"Mou L, Li G, Zhang L, Wang T, Jin Z (2016) Convolutional neural networks over tree structures for programming language processing. In: Proceedings of the AAAI conference on artificial intelligence, vol\u00a030","DOI":"10.1609\/aaai.v30i1.10139"},{"key":"10731_CR72","doi-asserted-by":"publisher","first-page":"2350001","DOI":"10.1142\/S2811032323500017","volume":"1","author":"PD Nagy","year":"2023","unstructured":"Nagy PD, Najafabadi MA, Davoudi H (2023) A survey on code representation. World Scientific Annual Review of Artificial Intelligence 1:2350001","journal-title":"World Scientific Annual Review of Artificial Intelligence"},{"key":"10731_CR73","doi-asserted-by":"crossref","unstructured":"Nair A, Roy A, Meinke K (2020) funcgnn: A graph neural network approach to program similarity. In: Proceedings of the 14th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM), pp 1\u201311","DOI":"10.1145\/3382494.3410675"},{"key":"10731_CR74","doi-asserted-by":"crossref","unstructured":"Nguyen HA, Nguyen TN, Dig D, Nguyen S, Tran H, Hilton M (2019) Graph-based mining of in-the-wild, fine-grained, semantic code change patterns. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE). IEEE, pp 819\u2013830","DOI":"10.1109\/ICSE.2019.00089"},{"issue":"6","key":"10731_CR75","doi-asserted-by":"publisher","first-page":"425","DOI":"10.1145\/1993316.1993548","volume":"46","author":"T Parr","year":"2011","unstructured":"Parr T, Fisher K (2011) Ll (*) the foundation of the antlr parser generator. ACM Sigplan Notices 46(6):425\u2013436","journal-title":"ACM Sigplan Notices"},{"key":"10731_CR76","doi-asserted-by":"crossref","unstructured":"Peng Y, Gao C, Li Z, Gao B, Lo D, Zhang Q, Lyu M (2022) Static inference meets deep learning: a hybrid type inference approach for python. In: Proceedings of the 44th international conference on software engineering, pp 2019\u20132030","DOI":"10.1145\/3510003.3510038"},{"key":"10731_CR77","doi-asserted-by":"crossref","unstructured":"Pennington J, Socher R, Manning CD (2014) Glove: Global vectors for word representation. In: Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP), pp 1532\u20131543","DOI":"10.3115\/v1\/D14-1162"},{"key":"10731_CR78","doi-asserted-by":"crossref","unstructured":"Pimentel T, Valvoda J, Maudslay RH, Zmigrod R, Williams A, Cotterell R (2020) Information-theoretic probing for linguistic structure. arXiv:2004.03061","DOI":"10.18653\/v1\/2020.acl-main.420"},{"key":"10731_CR79","doi-asserted-by":"crossref","unstructured":"Rajlich V (2014) Software evolution and maintenance. In: Future of software engineering proceedings, pp 133\u2013144","DOI":"10.1145\/2593882.2593893"},{"issue":"1","key":"10731_CR80","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1145\/2775051.2677009","volume":"50","author":"V Raychev","year":"2015","unstructured":"Raychev V, Vechev M, Krause A (2015) Predicting program properties from\" big code\". ACM SIGPLAN Notices 50(1):111\u2013124","journal-title":"ACM SIGPLAN Notices"},{"key":"10731_CR81","doi-asserted-by":"publisher","first-page":"842","DOI":"10.1162\/tacl_a_00349","volume":"8","author":"A Rogers","year":"2021","unstructured":"Rogers A, Kovaleva O, Rumshisky A (2021) A primer in bertology: What we know about how bert works. Trans Assoc Comput Linguist 8:842\u2013866","journal-title":"Trans Assoc Comput Linguist"},{"key":"10731_CR82","unstructured":"Rosenfeld JS, Rosenfeld A, Belinkov Y, Shavit N (2019) A constructive prediction of the generalization error across scales. arXiv:1909.12673"},{"issue":"115","key":"10731_CR83","first-page":"64","volume":"541","author":"CK Roy","year":"2007","unstructured":"Roy CK, Cordy JR (2007) A survey on software clone detection research. Queen\u2019s School of Computing TR 541(115):64\u201368","journal-title":"Queen\u2019s School of Computing TR"},{"issue":"7","key":"10731_CR84","doi-asserted-by":"publisher","first-page":"470","DOI":"10.1016\/j.scico.2009.02.007","volume":"74","author":"CK Roy","year":"2009","unstructured":"Roy CK, Cordy JR, Koschke R (2009) Comparison and evaluation of code clone detection techniques and tools: a qualitative approach. Sci Comput Program 74(7):470\u2013495","journal-title":"Sci Comput Program"},{"key":"10731_CR85","unstructured":"Roziere B, Gehring J, Gloeckle F, Sootla S, Gat I, Tan XE, Adi Y, Liu J, Sauvestre R, Remez T et\u00a0al. (2023) Code llama: open foundation models for code. arXiv:2308.12950"},{"issue":"5","key":"10731_CR86","doi-asserted-by":"publisher","first-page":"206","DOI":"10.1038\/s42256-019-0048-x","volume":"1","author":"C Rudin","year":"2019","unstructured":"Rudin C (2019) Stop explaining black box machine learning models for high stakes decisions and use interpretable models instead. Nat Mach Intell 1(5):206\u2013215","journal-title":"Nat Mach Intell"},{"key":"10731_CR87","unstructured":"Sanyal A, Chatterji V, Vyas N, Epstein B, Demir N, Corletti A (2021) Fix your models by fixing your datasets. arXiv:2112.07844"},{"key":"10731_CR88","unstructured":"Sculley D, Holt G, Golovin D, Davydov E, Phillips T, Ebner D, Chaudhary V, Young M, Crespo JF, Dennison D (2015) Hidden technical debt in machine learning systems. Adv Neural Inf Process Syst 28"},{"key":"10731_CR89","unstructured":"Sharir O, Peleg B, Shoham Y (2020) The cost of training nlp models: a concise overview. arXiv:2004.08900"},{"key":"10731_CR90","unstructured":"Sharma T, Kechagia M, Georgiou S, Tiwari R, Vats I, Moazen H, Sarro F (2021) A survey on machine learning techniques for source code analysis. arXiv:2110.09610"},{"key":"10731_CR91","doi-asserted-by":"crossref","unstructured":"Shido Y, Kobayashi Y, Yamamoto A, Miyamoto A, Matsumura T (2019) Automatic source code summarization with extended tree-lstm. In: 2019 International Joint Conference on Neural Networks (IJCNN). IEEE, pp 1\u20138","DOI":"10.1109\/IJCNN.2019.8851751"},{"key":"10731_CR92","first-page":"19339","volume":"33","author":"N Sohoni","year":"2020","unstructured":"Sohoni N, Dunnmon J, Angus G, Gu A, R\u00e9 C (2020) No subclass left behind: Fine-grained robustness in coarse-grained classification problems. Adv Neural Inf Process Syst 33:19339\u201319352","journal-title":"Adv Neural Inf Process Syst"},{"key":"10731_CR93","doi-asserted-by":"publisher","first-page":"111411","DOI":"10.1109\/ACCESS.2019.2931579","volume":"7","author":"X Song","year":"2019","unstructured":"Song X, Sun H, Wang X, Yan J (2019) A survey of automatic generation of source code comments: algorithms and techniques. IEEE Access 7:111411\u2013111428","journal-title":"IEEE Access"},{"key":"10731_CR94","doi-asserted-by":"crossref","unstructured":"Sun Z, Zhu Q, Mou L, Xiong Y, Li G, Zhang L (2019) A grammar-based structural cnn decoder for code generation. In: Proceedings of the AAAI conference on artificial intelligence vol 33, pp 7055\u20137062","DOI":"10.1609\/aaai.v33i01.33017055"},{"key":"10731_CR95","doi-asserted-by":"crossref","unstructured":"Svacina J, Simmons J, Cerny T (2020) Semantic code clone detection for enterprise applications. In: Proceedings of the 35th Annual ACM Symposium on Applied Computing, pp 129\u2013131","DOI":"10.1145\/3341105.3374117"},{"key":"10731_CR96","doi-asserted-by":"crossref","unstructured":"Svajlenko J, Islam JF, Keivanloo I, Roy CK, Mia MM (2014) Towards a big data curated benchmark of inter-project code clones. In: 2014 IEEE International conference on software maintenance and evolution. IEEE, pp 476\u2013480","DOI":"10.1109\/ICSME.2014.77"},{"key":"10731_CR97","doi-asserted-by":"crossref","unstructured":"Tai KS, Socher R, Manning CD (2015) Improved semantic representations from tree-structured long short-term memory networks. arXiv:1503.00075","DOI":"10.3115\/v1\/P15-1150"},{"key":"10731_CR98","doi-asserted-by":"crossref","unstructured":"Tambon F, Dakhel AM, Nikanjam A, Khomh F, Desmarais MC, Antoniol G (2024) Bugs in large language models generated code. arXiv:2403.08937","DOI":"10.1007\/s10664-025-10614-4"},{"key":"10731_CR99","doi-asserted-by":"crossref","unstructured":"Tang N, Chen M, Ning Z, Bansal A, Huang Y, McMillan C, Li TJJ (2024) A study on developer behaviors for validating and repairing llm-generated code using eye tracking and ide actions. arXiv:2405.16081","DOI":"10.1109\/VL\/HCC60511.2024.00015"},{"key":"10731_CR100","doi-asserted-by":"publisher","first-page":"94","DOI":"10.1016\/j.infsof.2017.11.008","volume":"96","author":"H Tong","year":"2018","unstructured":"Tong H, Liu B, Wang S (2018) Software defect prediction using stacked denoising autoencoders and two-stage ensemble learning. Inf Softw Technol 96:94\u2013111","journal-title":"Inf Softw Technol"},{"key":"10731_CR101","doi-asserted-by":"crossref","unstructured":"Troshin S, Chirkova N (2022) Probing pretrained models of source code. arXiv:2202.08975","DOI":"10.18653\/v1\/2022.blackboxnlp-1.31"},{"key":"10731_CR102","doi-asserted-by":"crossref","unstructured":"Tufano M, Watson C, Bavota G, Di\u00a0Penta M, White M, Poshyvanyk D (2018) Deep learning similarities from different representations of source code. In: Proceedings of the 15th international conference on mining software repositories, pp 542\u2013553","DOI":"10.1145\/3196398.3196431"},{"issue":"4","key":"10731_CR103","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3340544","volume":"28","author":"M Tufano","year":"2019","unstructured":"Tufano M, Watson C, Bavota G, Penta MD, White M, Poshyvanyk D (2019) An empirical study on learning bug-fixing patches in the wild via neural machine translation. ACM Trans Softw Eng Methodol (TOSEM) 28(4):1\u201329","journal-title":"ACM Trans Softw Eng Methodol (TOSEM)"},{"key":"10731_CR104","unstructured":"Vandaele R, Kang B, De\u00a0Bie T, Saeys Y (2022) The curse revisited: When are distances informative for the ground truth in noisy high-dimensional data? In: International conference on artificial intelligence and statistics. PMLR, pp 2158\u20132172"},{"key":"10731_CR105","unstructured":"Vilone G, Longo L (2020) Explainable artificial intelligence: a systematic review. arXiv:2006.00093"},{"key":"10731_CR106","doi-asserted-by":"crossref","unstructured":"Wan Y, Zhao Z, Yang M, Xu G, Ying H, Wu J, Yu PS (2018) Improving automatic source code summarization via deep reinforcement learning. In: Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering, pp 397\u2013407","DOI":"10.1145\/3238147.3238206"},{"key":"10731_CR107","doi-asserted-by":"crossref","unstructured":"Wan Y, Zhao W, Zhang H, Sui Y, Xu G, Jin H (2022) What do they capture? a structural analysis of pre-trained language models for source code. In: Proceedings of the 44th international conference on software engineering, pp 2377\u20132388","DOI":"10.1145\/3510003.3510050"},{"key":"10731_CR108","unstructured":"Wang X, Wang Y, Mi F, Zhou P, Wan Y, Liu X, Li L, Wu H, Liu J, Jiang X (2021a) Syncobert: Syntax-guided multi-modal contrastive pre-training for code representation. arXiv:2108.04556"},{"key":"10731_CR109","doi-asserted-by":"crossref","unstructured":"Wang Y, Wang W, Joty S, Hoi SC (2021b) Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. arXiv:2109.00859","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"10731_CR110","doi-asserted-by":"crossref","unstructured":"Wang Y, Le H, Gotmare AD, Bui ND, Li J, Hoi SC (2023) Codet5+: Open code large language models for code understanding and generation. arXiv:2305.07922","DOI":"10.18653\/v1\/2023.emnlp-main.68"},{"key":"10731_CR111","unstructured":"Wei B, Li G, Xia X, Fu Z, Jin Z (2019) Code generation as a dual task of code summarization. Adv Neural Inf Process Syst 32"},{"key":"10731_CR112","doi-asserted-by":"crossref","unstructured":"White M, Tufano M, Vendome C, Poshyvanyk D (2016) Deep learning code fragments for code clone detection. In: Proceedings of the 31st IEEE\/ACM international conference on automated software engineering, pp 87\u201398","DOI":"10.1145\/2970276.2970326"},{"key":"10731_CR113","doi-asserted-by":"crossref","unstructured":"Wolf T, Debut L, Sanh V, Chaumond J, Delangue C, Moi A, Cistac P, Rault T, Louf R, Funtowicz M et\u00a0al. (2019) Huggingface\u2019s transformers: State-of-the-art natural language processing. arXiv:1910.03771","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"10731_CR114","doi-asserted-by":"crossref","unstructured":"Xu Z, Zhang X, Chen L, Pei K, Xu B (2016) Python probabilistic type inference with natural language support. In: Proceedings of the 2016 24th ACM SIGSOFT international symposium on foundations of software engineering, pp 607\u2013618","DOI":"10.1145\/2950290.2950343"},{"key":"10731_CR115","doi-asserted-by":"crossref","unstructured":"Yaghoobzadeh Y, Kann K, Hazen TJ, Agirre E, Sch\u00fctze H (2019) Probing for semantic classes: diagnosing the meaning content of word embeddings. arXiv:1906.03608","DOI":"10.18653\/v1\/P19-1574"},{"issue":"10s","key":"10731_CR116","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3505243","volume":"54","author":"Y Yang","year":"2022","unstructured":"Yang Y, Xia X, Lo D, Grundy J (2022) A survey on deep learning for software engineering. ACM Comput Surv (CSUR) 54(10s):1\u201373","journal-title":"ACM Comput Surv (CSUR)"},{"key":"10731_CR117","doi-asserted-by":"crossref","unstructured":"Yin X, Ni C, Wang S, Li Z, Zeng L, Yang X (2024) Thinkrepair: Self-directed automated program repair. In: Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis, pp 1274\u20131286","DOI":"10.1145\/3650212.3680359"},{"key":"10731_CR118","unstructured":"Zhang C, Bengio S, Hardt M, Recht B, Vinyals O (2016) Understanding deep learning requires rethinking generalization. arXiv:1611.03530"},{"key":"10731_CR119","doi-asserted-by":"crossref","unstructured":"Zhang J, Wang X, Zhang H, Sun H, Wang K, Liu X (2019a) A novel neural source code representation based on abstract syntax tree. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE). IEEE, pp 783\u2013794","DOI":"10.1109\/ICSE.2019.00086"},{"key":"10731_CR120","doi-asserted-by":"crossref","unstructured":"Zhang T, Gao C, Ma L, Lyu M, Kim M (2019b) An empirical study of common challenges in developing deep learning applications. In: 2019 IEEE 30th International Symposium on Software Reliability Engineering (ISSRE). IEEE, pp 104\u2013115","DOI":"10.1109\/ISSRE.2019.00020"},{"issue":"3","key":"10731_CR121","doi-asserted-by":"publisher","first-page":"471","DOI":"10.3390\/sym14030471","volume":"14","author":"C Zhang","year":"2022","unstructured":"Zhang C, Wang J, Zhou Q, Xu T, Tang K, Gui H, Liu F (2022) A survey of automatic source code summarization. Symmetry 14(3):471","journal-title":"Symmetry"},{"issue":"2","key":"10731_CR122","first-page":"1","volume":"33","author":"Q Zhang","year":"2023","unstructured":"Zhang Q, Fang C, Ma Y, Sun W, Chen Z (2023a) A survey of learning-based automated program repair. ACM Trans Softw Eng Methodol 33(2):1\u201369","journal-title":"ACM Trans Softw Eng Methodol"},{"key":"10731_CR123","unstructured":"Zhang Y, Li Y, Cui L, Cai D, Liu L, Fu T, Huang X, Zhao E, Zhang Y, Chen Y et\u00a0al. (2023b) Siren\u2019s song in the ai ocean: a survey on hallucination in large language models. arXiv:2309.01219"},{"key":"10731_CR124","unstructured":"Zhang D, Ahmad W, Tan M, Ding H, Nallapati R, Roth D, Ma X, Xiang B (2024) Code representation learning at scale. arXiv:2402.01935"},{"key":"10731_CR125","unstructured":"Zhao WX, Zhou K, Li J, Tang T, Wang X, Hou Y, Min Y, Zhang B, Zhang J, Dong Z et\u00a0al. (2023) A survey of large language models. arXiv:2303.18223"},{"issue":"2","key":"10731_CR126","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3639372","volume":"15","author":"H Zhao","year":"2024","unstructured":"Zhao H, Chen H, Yang F, Liu N, Deng H, Cai H, Wang S, Yin D, Du M (2024) Explainability for large language models: a survey. ACM Trans Intell Syst Technol 15(2):1\u201338","journal-title":"ACM Trans Intell Syst Technol"},{"key":"10731_CR127","unstructured":"Zhou X, Liang P, Zhang B, Li Z, Ahmad A, Shahin M, Waseem M (2023) On the concerns of developers when using github copilot. arXiv:2311.01020"},{"key":"10731_CR128","doi-asserted-by":"crossref","unstructured":"Zhou Y, Tu F, Sha K, Ding J, Chen H (2024) A survey on data quality dimensions and tools for machine learning. arXiv:2406.19614","DOI":"10.1109\/AITest62860.2024.00023"},{"key":"10731_CR129","doi-asserted-by":"crossref","unstructured":"Zhu Q, Sun Z, Xiao Ya, Zhang W, Yuan K, Xiong Y, Zhang L (2021) A syntax-guided edit decoder for neural program repair. In: Proceedings of the 29th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering, pp 341\u2013353","DOI":"10.1145\/3468264.3468544"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10731-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10731-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10731-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,5]],"date-time":"2025-11-05T02:41:47Z","timestamp":1762310507000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10731-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,30]]},"references-count":129,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2025,12]]}},"alternative-id":["10731"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10731-0","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2025,9,30]]},"assertion":[{"value":"9 December 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 September 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 September 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Not applicable to this study.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical Approval"}},{"value":"Not applicable to this study.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed Consent"}},{"value":"The authors declare that they have no conflict of interest.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interests"}},{"value":"Not applicable to this study.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Clinical Trial Number"}}],"article-number":"169"}}