{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,16]],"date-time":"2026-10-16T21:07:13Z","timestamp":1792184833231,"version":"4.4.0"},"publisher-location":"New York, NY, USA","reference-count":97,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T00:00:00Z","timestamp":1712880000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000781","name":"European Research Council","doi-asserted-by":"publisher","award":["851720"],"award-info":[{"award-number":["851720"]}],"id":[{"id":"10.13039\/501100000781","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Italian ministry of research","award":["2020W3A5FY"],"award-info":[{"award-number":["2020W3A5FY"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,12]]},"DOI":"10.1145\/3597503.3639174","type":"proceedings-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T16:43:26Z","timestamp":1712940206000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":39,"title":["Evaluating Code Summarization Techniques: A New Metric and an Empirical Characterization"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7965-7712","authenticated-orcid":false,"given":"Antonio","family":"Mastropaolo","sequence":"first","affiliation":[{"name":"SEART (Software Analytics Research Team), Universita\u00a0 della Svizzera Italiana, Lugano, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0251-4473","authenticated-orcid":false,"given":"Matteo","family":"Ciniselli","sequence":"additional","affiliation":[{"name":"SEART (Software Analytics Research Team), Universita\u00a0 della Svizzera Italiana, Lugano, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0340-9747","authenticated-orcid":false,"given":"Massimiliano","family":"Di Penta","sequence":"additional","affiliation":[{"name":"Department of Engineering, University of Sannio, Benevento, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2216-3148","authenticated-orcid":false,"given":"Gabriele","family":"Bavota","sequence":"additional","affiliation":[{"name":"SEART (Software Analytics Research Team), Universita\u00a0 della Svizzera Italiana, Lugano, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. CC-News Dataset. https:\/\/commoncrawl.org\/2016\/10\/news-dataset-available\/."},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. Replication Package https:\/\/github.com\/antonio-mastropaolo\/code-summarization-metric\/tree\/main."},{"key":"e_1_3_2_1_3_1","unstructured":"[n. d.]. Stories Dataset. https:\/\/github.com\/tensorflow\/models\/tree\/archive\/research\/lm_commonsense#1-download-data-files."},{"key":"e_1_3_2_1_4_1","first-page":"1","article-title":"Automated Documentation of Android Apps","volume":"45","author":"Aghajani Emad","year":"2019","unstructured":"Emad Aghajani, Gabriele Bavota, Mario Linares-V\u00e1squez, and Michele Lanza. 2019. Automated Documentation of Android Apps. IEEE Transactions on Software Engineering 45, 1 (2019), 1--13.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_5_1","volume-title":"Conference of the North American","author":"Ahmad Wasi Uddin","unstructured":"Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, and Kai-Wei Chang. 2021. Unified Pre-training for Program Understanding and Generation. In Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT. 2655--2668."},{"key":"e_1_3_2_1_6_1","volume-title":"International Conference on Machine Learning (ICML).","author":"Allamanis Miltiadis","year":"2016","unstructured":"Miltiadis Allamanis, Hao Peng, and Charles Sutton. 2016. A Convolutional Attention Network for Extreme Summarization of Source Code. In International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_7_1","volume-title":"code2seq: Generating sequences from structured representations of code. arXiv preprint arXiv:1808.01400","author":"Alon Uri","year":"2018","unstructured":"Uri Alon, Shaked Brody, Omer Levy, and Eran Yahav. 2018. code2seq: Generating sequences from structured representations of code. arXiv preprint arXiv:1808.01400 (2018)."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the ACL Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and\/or Summarization. Association for Computational Linguistics","author":"Banerjee Satanjeev","year":"2005","unstructured":"Satanjeev Banerjee and Alon Lavie. 2005. METEOR: An Automatic Metric for MT Evaluation with Improved Correlation with Human Judgments. In Proceedings of the ACL Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and\/or Summarization. Association for Computational Linguistics, Ann Arbor, Michigan, 65--72. https:\/\/aclanthology.org\/W05-0909"},{"key":"e_1_3_2_1_9_1","series-title":"Series B (Methodological) 57, 1","volume-title":"Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing. Journal of the Royal Statistical Society","author":"Benjamini Yoav","year":"1995","unstructured":"Yoav Benjamini and Yosef Hochberg. 1995. Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing. Journal of the Royal Statistical Society. Series B (Methodological) 57, 1 (1995)."},{"key":"e_1_3_2_1_10_1","volume-title":"Noah Constant, Mario Guajardo-Cespedes, Steve Yuan, Chris Tar, et al.","author":"Cer Daniel","year":"2018","unstructured":"Daniel Cer, Yinfei Yang, Sheng-yi Kong, Nan Hua, Nicole Limtiaco, Rhomni St John, Noah Constant, Mario Guajardo-Cespedes, Steve Yuan, Chris Tar, et al. 2018. Universal sentence encoder. arXiv preprint arXiv:1803.11175 (2018)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2019.03.010"},{"key":"e_1_3_2_1_12_1","volume-title":"Learning a similarity metric discriminatively, with application to face verification. In 2005 IEEE computer society conference on computer vision and pattern recognition (CVPR'05)","author":"Chopra Sumit","unstructured":"Sumit Chopra, Raia Hadsell, and Yann LeCun. 2005. Learning a similarity metric discriminatively, with application to face verification. In 2005 IEEE computer society conference on computer vision and pattern recognition (CVPR'05), Vol. 1. IEEE, 539--546."},{"key":"e_1_3_2_1_13_1","volume-title":"Supervised learning of universal sentence representations from natural language inference data. arXiv preprint arXiv:1705.02364","author":"Conneau Alexis","year":"2017","unstructured":"Alexis Conneau, Douwe Kiela, Holger Schwenk, Loic Barrault, and Antoine Bordes. 2017. Supervised learning of universal sentence representations from natural language inference data. arXiv preprint arXiv:1705.02364 (2017)."},{"key":"e_1_3_2_1_14_1","volume-title":"Practical Nonparametric Statistics","author":"Conover W. J.","unstructured":"W. J. Conover. 1998. Practical Nonparametric Statistics (3rd edition ed.). Wiley.","edition":"3"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/157710.157715"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 23rd Annual International Conference on Design of Communication: Documenting & Designing for Pervasive Information (SIGDOC '05)","author":"de Souza Sergio Cozzetti B.","unstructured":"Sergio Cozzetti B. de Souza, Nicolas Anquetil, and K\u00e1thia M. de Oliveira. 2005. A Study of the Documentation Essential to Software Maintenance. In Proceedings of the 23rd Annual International Conference on Design of Communication: Documenting & Designing for Pervasive Information (SIGDOC '05). 68--75."},{"key":"e_1_3_2_1_17_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Conference of the North American","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT. 4171--4186."},{"key":"e_1_3_2_1_18_1","volume-title":"14th Working Conference on Reverse Engineering, WCRE. 70--79","author":"Fluri Beat","unstructured":"Beat Fluri, Michael Wursch, and Harald C. Gall. 2007. Do Code and Comments Co-Evolve? On the Relation between Source Code and Comment Changes. In 14th Working Conference on Reverse Engineering, WCRE. 70--79."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-009-9075-x"},{"key":"e_1_3_2_1_20_1","volume-title":"An Efficient Method of Supervised Contrastive Learning for Natural Language Understanding. In 7th International Conference on Computer and Communications (ICCC). 1698--1704","author":"Gao Haifeng","year":"2021","unstructured":"Haifeng Gao, Bin Dong, Yongwei Zhangand Tianxiong Xiao, Shanshan Jiang, and Yuan Dong. 2021. An Efficient Method of Supervised Contrastive Learning for Natural Language Understanding. In 7th International Conference on Computer and Communications (ICCC). 1698--1704."},{"key":"e_1_3_2_1_21_1","unstructured":"Aaron Gokaslan and Vanya Cohen. 2019. OpenWebText Corpus. http:\/\/Skylion007.github.io\/OpenWebTextCorpus."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 7th Linguistic Annotation Workshop and Interoperability with Discourse. 33--41","author":"Graham Yvette","year":"2013","unstructured":"Yvette Graham, Timothy Baldwin, Alistair Moffat, and Justin Zobel. 2013. Continuous measurement scales in human evaluation of machine translation. In Proceedings of the 7th Linguistic Annotation Workshop and Interoperability with Discourse. 33--41."},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering. 746--757","author":"Gros David","year":"2021","unstructured":"David Gros, Hariharan Sezhiyan, Prem Devanbu, and Zhou Yu. 2021. Code to Comment \"Translation\": Data, Metrics, Baselining & Evaluation. In Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering. 746--757."},{"key":"e_1_3_2_1_24_1","volume-title":"GraphCodeBERT: Pre-training Code Representations with Data Flow. In 9th International Conference on Learning Representations, ICLR","author":"Guo Daya","year":"2021","unstructured":"Daya Guo, Shuo Ren, Shuai Lu, Zhangyin Feng, Duyu Tang, Shujie Liu, Long Zhou, Nan Duan, Alexey Svyatkovskiy, Shengyu Fu, Michele Tufano, Shao Kun Deng, Colin B. Clement, Dawn Drain, Neel Sundaresan, Jian Yin, Daxin Jiang, and Ming Zhou. 2021. GraphCodeBERT: Pre-training Code Representations with Data Flow. In 9th International Conference on Learning Representations, ICLR 2021."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2006.100"},{"key":"e_1_3_2_1_26_1","volume-title":"On the Use of Automated Text Summarization Techniques for Summarizing Source Code. In 17th Working Conference on Reverse Engineering, WCRE. 35--44","author":"Haiduc Sonia","year":"2010","unstructured":"Sonia Haiduc, Jairo Aponte, Laura Moreno, and Andrian Marcus. 2010. On the Use of Automated Text Summarization Techniques for Summarizing Source Code. In 17th Working Conference on Reverse Engineering, WCRE. 35--44."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","DOI":"10.1002\/0471650129","volume-title":"Jaccard similarity coefficient). Dictionary of Bioinformatics and Computational Biology","author":"Hancock John M","year":"2004","unstructured":"John M Hancock. 2004. Jaccard distance (Jaccard index, Jaccard similarity coefficient). Dictionary of Bioinformatics and Computational Biology (2004)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524610.3527909"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Sakib Haque Alexander LeClair Lingfei Wu and Collin McMillan. 2020. Improved Automatic Summarization of Subroutines via Attention to File Context. arXiv:2004.04881 [cs.SE]","DOI":"10.1145\/3379597.3387449"},{"key":"e_1_3_2_1_30_1","volume-title":"with contributions from Charles Dupont, and many others","author":"Harrell Frank E","year":"2020","unstructured":"Frank E Harrell Jr, with contributions from Charles Dupont, and many others. 2020. Hmisc: Harrell Miscellaneous. https:\/\/CRAN.R-project.org\/package=Hmisc R package version 4.3-1."},{"key":"e_1_3_2_1_31_1","volume-title":"Correlating Automated and Human Evaluation of Code Documentation Generation Quality. 31, 4","author":"Hu Xing","year":"2022","unstructured":"Xing Hu, Qiuyuan Chen, Haoye Wang, Xin Xia, David Lo, and Thomas Zimmermann. 2022. Correlating Automated and Human Evaluation of Code Documentation Generation Quality. 31, 4 (2022)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09730-9"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Junjie Huang Duyu Tang Wanjun Zhong Shuai Lu Linjun Shou Ming Gong Daxin Jiang and Nan Duan. 2021. WhiteningBERT: An Easy Unsupervised Sentence Embedding Approach. In Findings of the Association for Computational Linguistics: EMNLP. 238--244.","DOI":"10.18653\/v1\/2021.findings-emnlp.23"},{"key":"e_1_3_2_1_34_1","volume-title":"Towards automatically generating block comments for code snippets. Inf. Softw. Technol. 127","author":"Huang Yuan","year":"2020","unstructured":"Yuan Huang, Shaohao Huang, Huanchao Chen, Xiangping Chen, Zibin Zheng, Xiapu Luo, Nan Jia, Xinyu Hu, and Xiaocong Zhou. 2020. Towards automatically generating block comments for code snippets. Inf. Softw. Technol. 127 (2020)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","unstructured":"Peter J Huber. 1992. Robust estimation of a location parameter. In Breakthroughs in statistics: Methodology and distribution. 492--518.","DOI":"10.1007\/978-1-4612-4380-9_35"},{"key":"e_1_3_2_1_36_1","volume-title":"Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436","author":"Husain Hamel","year":"2019","unstructured":"Hamel Husain, Ho-Hsiang Wu, Tiferet Gazit, Miltiadis Allamanis, and Marc Brockschmidt. 2019. Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436 (2019)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"e_1_3_2_1_38_1","volume-title":"Principal Component Analysis","author":"Jolliffe Ian T.","unstructured":"Ian T. Jolliffe. 1986. Principal Component Analysis. Springer."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Ninus Khamis Ren\u00e9 Witte and Juergen Rilling. 2010. Automatic Quality Assessment of Source Code Comments: The JavadocMiner. In Natural Language Processing and Information Systems Christina J. Hopfe Yacine Rezgui Elisabeth M\u00e9tais Alun Preece and Haijiang Li (Eds.). 68--79.","DOI":"10.1007\/978-3-642-13881-2_7"},{"key":"e_1_3_2_1_40_1","volume-title":"Information theory and statistics","author":"Kullback Solomon","unstructured":"Solomon Kullback. 1997. Information theory and statistics. Courier Corporation."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00087"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00087"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1394"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"e_1_3_2_1_45_1","volume-title":"Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81.","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81."},{"key":"e_1_3_2_1_46_1","volume-title":"30th IEEE\/ACM International Conference on Automated Software Engineering, ASE. 36--41","author":"Linares-V\u00e1squez M.","unstructured":"M. Linares-V\u00e1squez, B. Li, C. Vendome, and D. Poshyvanyk. 2015. How do Developers Document Database Usages in Source Code?. In 30th IEEE\/ACM International Conference on Automated Software Engineering, ASE. 36--41."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1907.11692"},{"key":"e_1_3_2_1_48_1","volume-title":"Automatic Detection of Outdated Comments During Code Changes. In 2018 IEEE 42nd Annual Computer Software and Applications Conference (COMPSAC)","volume":"01","author":"Liu Zhiyong","year":"2018","unstructured":"Zhiyong Liu, Huanchao Chen, Xiangping Chen, Xiaonan Luo, and Fan Zhou. 2018. Automatic Detection of Outdated Comments During Code Changes. In 2018 IEEE 42nd Annual Computer Software and Applications Conference (COMPSAC), Vol. 01. 154--163."},{"key":"e_1_3_2_1_49_1","volume-title":"Decoupled Weight Decay Regularization. In 7th International Conference on Learning Representations, ICLR.","author":"Loshchilov Ilya","year":"2019","unstructured":"Ilya Loshchilov and Frank Hutter. 2019. Decoupled Weight Decay Regularization. In 7th International Conference on Learning Representations, ICLR."},{"key":"e_1_3_2_1_50_1","volume-title":"Codexglue: A machine learning benchmark dataset for code understanding and generation. arXiv preprint arXiv:2102.04664","author":"Lu Shuai","year":"2021","unstructured":"Shuai Lu, Daya Guo, Shuo Ren, Junjie Huang, Alexey Svyatkovskiy, Ambrosio Blanco, Colin Clement, Dawn Drain, Daxin Jiang, Duyu Tang, et al. 2021. Codexglue: A machine learning benchmark dataset for code understanding and generation. arXiv preprint arXiv:2102.04664 (2021)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.109265"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2465386"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2013.6613830"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.434"},{"key":"e_1_3_2_1_55_1","volume-title":"Questionnaire Design, Interviewing and Attitude Measurement","author":"Oppenheim A. N.","unstructured":"A. N. Oppenheim. 1992. Questionnaire Design, Interviewing and Attitude Measurement. Pinter Publishers."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21362"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21362"},{"key":"e_1_3_2_1_58_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W15-3049"},{"key":"e_1_3_2_1_61_1","volume-title":"An Exploratory Study on Self-Admitted Technical Debt. In 2014 IEEE International Conference on Software Maintenance and Evolution. 91--100","author":"Potdar Aniket","year":"2014","unstructured":"Aniket Potdar and Emad Shihab. 2014. An Exploratory Study on Self-Admitted Technical Debt. In 2014 IEEE International Conference on Software Maintenance and Evolution. 91--100."},{"key":"e_1_3_2_1_62_1","volume-title":"R: A Language and Environment for Statistical Computing. https:\/\/www.R-project.org\/","author":"Team R Core","year":"2020","unstructured":"R Core Team. 2020. R: A Language and Environment for Statistical Computing. https:\/\/www.R-project.org\/"},{"key":"e_1_3_2_1_63_1","article-title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","volume":"21","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. J. Mach. Learn. Res. 21 (2020), 140:1--140:67.","journal-title":"J. Mach. Learn. Res."},{"key":"e_1_3_2_1_64_1","volume-title":"15th IEEE International Working Conference on Source Code Analysis and Manipulation, SCAM. 81--90","author":"Rahman M. M.","unstructured":"M. M. Rahman, C. K. Roy, and I. Keivanloo. 2015. Recommending insightful comments for source code using crowdsourced knowledge. In 15th IEEE International Working Conference on Source Code Analysis and Manipulation, SCAM. 81--90."},{"key":"e_1_3_2_1_65_1","volume-title":"Proc. of the 1st instructional conf. on machine learning","volume":"242","author":"Juan","unstructured":"Juan Ramos et al. 2003. Using tf-idf to determine word relevance in document queries. In Proc. of the 1st instructional conf. on machine learning, Vol. 242. 29--48."},{"key":"e_1_3_2_1_66_1","volume-title":"Sentence-bert: Sentence embeddings using siamese bert-networks. arXiv preprint arXiv:1908.10084","author":"Reimers Nils","year":"2019","unstructured":"Nils Reimers and Iryna Gurevych. 2019. Sentence-bert: Sentence embeddings using siamese bert-networks. arXiv preprint arXiv:1908.10084 (2019)."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.13"},{"key":"e_1_3_2_1_68_1","volume-title":"29th ACM Joint Meeting on European Software Engineering Conference and the ACM\/SIGSOFT Symposium on the Foundations of Software Engineering, ESEC-FSE. 1105--1116.","author":"Roy Devjeet","unstructured":"Devjeet Roy, Sarah Fakhoury, and Venera Arnaoudova. 2021. Reassessing Automatic Evaluation Metrics for Code Summarization Tasks. In 29th ACM Joint Meeting on European Software Engineering Conference and the ACM\/SIGSOFT Symposium on the Foundations of Software Engineering, ESEC-FSE. 1105--1116."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2109.14250"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"e_1_3_2_1_71_1","volume-title":"MPNet: Masked and Permuted Pre-training for Language Understanding. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems.","author":"Song Kaitao","year":"2020","unstructured":"Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, and Tie-Yan Liu. 2020. MPNet: Masked and Permuted Pre-training for Language Understanding. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2010.95"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/1985793.1985808"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389258"},{"key":"e_1_3_2_1_75_1","volume-title":"Proc. of the 21st IEEE International Conference on Program Comprehension (ICPC). 83--92","author":"Steidl D.","unstructured":"D. Steidl, B. Hummel, and E. Juergens. 2013. Quality analysis of source code comments. In Proc. of the 21st IEEE International Conference on Program Comprehension (ICPC). 83--92."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2013.6613836"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/1294261.1294276"},{"key":"e_1_3_2_1_78_1","volume-title":"Proceedings of HotOS'07: 11th Workshop on Hot Topics in Operating Systems, Galen C. Hunt (Ed.). USENIX Association.","author":"Tan Lin","year":"2007","unstructured":"Lin Tan, Ding Yuan, and Yuanyuan Zhou. 2007. HotComments: How to Make Program Comments More Useful?. In Proceedings of HotOS'07: 11th Workshop on Hot Topics in Operating Systems, Galen C. Hunt (Ed.). USENIX Association."},{"key":"e_1_3_2_1_79_1","volume-title":"2012 IEEE Fifth International Conference on Software Testing, Verification and Validation. 260--269","author":"Tan Shin Hwei","unstructured":"Shin Hwei Tan, Darko Marinov, Lin Tan, and Gary T. Leavens. 2012. @tComment: Testing Javadoc Comments to Detect Comment-Code Inconsistencies. In 2012 IEEE Fifth International Conference on Software Testing, Verification and Validation. 260--269."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.6171"},{"key":"e_1_3_2_1_81_1","volume-title":"30th Advances in Neural Information Processing Systems. 5998--6008.","author":"Vaswani Ashish","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is All you Need. In 30th Advances in Neural Information Processing Systems. 5998--6008."},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"crossref","unstructured":"W. N. Venables and B. D. Ripley. 2002. Modern Applied Statistics with S (fourth ed.). Springer New York. http:\/\/www.stats.ox.ac.uk\/pub\/MASS4 ISBN 0-387-95457-0.","DOI":"10.1007\/978-0-387-21706-2_14"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2957424"},{"key":"e_1_3_2_1_84_1","volume-title":"Nghi D. Q. Bui, Junnan Li, and Steven C. H. Hoi.","author":"Wang Yue","year":"2023","unstructured":"Yue Wang, Hung Le, Akhilesh Deepak Gotmare, Nghi D. Q. Bui, Junnan Li, and Steven C. H. Hoi. 2023. CodeT5+: Open Code Large Language Models for Code Understanding and Generation. arXiv:2305.07922 [cs.CL]"},{"key":"e_1_3_2_1_85_1","volume-title":"Semi-supervised clustering with contrastive learning for discovering new intents. arXiv preprint arXiv:2201.07604","author":"Wei Feng","year":"2022","unstructured":"Feng Wei, Zhenbo Chen, Zhenghong Hao, Fengxin Yang, Hua Wei, Bing Han, and Sheng Guo. 2022. Semi-supervised clustering with contrastive learning for discovering new intents. arXiv preprint arXiv:2201.07604 (2022)."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2019.00019"},{"key":"e_1_3_2_1_87_1","volume-title":"Evolution and Reengineering (SANER), 2015 IEEE 22nd International Conference on. 380--389","author":"Wong Edmund","year":"2015","unstructured":"Edmund Wong, Taiyue Liu, and Lin Tan. 2015. CloCom: Mining existing source code for automatic comment generation. In Software Analysis, Evolution and Reengineering (SANER), 2015 IEEE 22nd International Conference on. 380--389."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2013.6693113"},{"key":"e_1_3_2_1_89_1","volume-title":"Proceedings of the 5th International Conference on Software Engineering (ICSE '81)","author":"Woodfield S. N.","unstructured":"S. N. Woodfield, H. E. Dunsmore, and V. Y. Shen. 1981. The Effect of Modularization and Comments on Program Comprehension. In Proceedings of the 5th International Conference on Software Engineering (ICSE '81). 215--223."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2734091"},{"key":"e_1_3_2_1_91_1","volume-title":"Graph2seq: Graph to sequence learning with attention-based neural networks. arXiv preprint arXiv:1804.00823","author":"Xu Kun","year":"2018","unstructured":"Kun Xu, Lingfei Wu, Zhiguo Wang, Yansong Feng, Michael Witbrock, and Vadim Sheinin. 2018. Graph2seq: Graph to sequence learning with attention-based neural networks. arXiv preprint arXiv:1804.00823 (2018)."},{"key":"e_1_3_2_1_92_1","volume-title":"Le","author":"Yang Zhilin","year":"2019","unstructured":"Zhilin Yang, Zihang Dai, Yiming Yang, Jaime G. Carbonell, Ruslan Salakhutdinov, and Quoc V. Le. 2019. XLNet: Generalized Autoregressive Pretraining for Language Understanding. In Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems. 5754--5764."},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380383"},{"key":"e_1_3_2_1_94_1","volume-title":"PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization. In 37th International Conference on Machine Learning, ICML. 11328--11339","author":"Zhang Jingqing","unstructured":"Jingqing Zhang, Yao Zhao, Mohammad Saleh, and Peter J. Liu. 2020. PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization. In 37th International Conference on Machine Learning, ICML. 11328--11339."},{"key":"e_1_3_2_1_95_1","volume-title":"Bertscore: Evaluating text generation with bert. arXiv preprint arXiv:1904.09675","author":"Zhang Tianyi","year":"2019","unstructured":"Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q Weinberger, and Yoav Artzi. 2019. Bertscore: Evaluating text generation with bert. arXiv preprint arXiv:1904.09675 (2019)."},{"key":"e_1_3_2_1_96_1","volume-title":"BERTScore: Evaluating Text Generation with BERT. In 8th International Conference on Learning Representations, ICLR.","author":"Zhang Tianyi","year":"2020","unstructured":"Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. 2020. BERTScore: Evaluating Text Generation with BERT. In 8th International Conference on Learning Representations, ICLR."},{"key":"e_1_3_2_1_97_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.11"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639174","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3639174","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:49:12Z","timestamp":1750286952000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639174"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,12]]},"references-count":97,"alternative-id":["10.1145\/3597503.3639174","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3639174","relation":{},"subject":[],"published":{"date-parts":[[2024,4,12]]},"assertion":[{"value":"2024-04-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}