{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T11:56:26Z","timestamp":1775822186627,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,16]],"date-time":"2022-05-16T00:00:00Z","timestamp":1652659200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSF","award":["CCF-2100035"],"award-info":[{"award-number":["CCF-2100035"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,16]]},"DOI":"10.1145\/3524610.3527909","type":"proceedings-article","created":{"date-parts":[[2022,10,20]],"date-time":"2022-10-20T15:19:30Z","timestamp":1666279170000},"page":"36-47","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":85,"title":["Semantic similarity metrics for evaluating source code summarization"],"prefix":"10.1145","author":[{"given":"Sakib","family":"Haque","sequence":"first","affiliation":[{"name":"University of Notre Dame"}]},{"given":"Zachary","family":"Eberhart","sequence":"additional","affiliation":[{"name":"University of Notre Dame"}]},{"given":"Aakash","family":"Bansal","sequence":"additional","affiliation":[{"name":"University of Notre Dame"}]},{"given":"Collin","family":"McMillan","sequence":"additional","affiliation":[{"name":"University of Notre Dame"}]}],"member":"320","published-online":{"date-parts":[[2022,10,20]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"416","volume-title":"An empirical study on evolution of api documentation,\" in International Conference on Fundamental Approaches To Software Engineering","author":"Shi L.","year":"2011","unstructured":"L. Shi , H. Zhong , T. Xie , and M. Li , \" An empirical study on evolution of api documentation,\" in International Conference on Fundamental Approaches To Software Engineering . Springer , 2011 , pp. 416 -- 431 . L. Shi, H. Zhong, T. Xie, and M. Li, \"An empirical study on evolution of api documentation,\" in International Conference on Fundamental Approaches To Software Engineering. Springer, 2011, pp. 416--431."},{"key":"e_1_3_2_1_2_1","first-page":"803","volume-title":"Detecting api documentation errors,\" in Proceedings of the 2013 ACM SIGPLAN international conference on Object oriented programming systems languages & applications","author":"Zhong H.","year":"2013","unstructured":"H. Zhong and Z. Su , \" Detecting api documentation errors,\" in Proceedings of the 2013 ACM SIGPLAN international conference on Object oriented programming systems languages & applications , 2013 , pp. 803 -- 816 . H. Zhong and Z. Su, \"Detecting api documentation errors,\" in Proceedings of the 2013 ACM SIGPLAN international conference on Object oriented programming systems languages & applications, 2013, pp. 803--816."},{"key":"e_1_3_2_1_3_1","first-page":"147","volume-title":"ACM","author":"Kramer D.","year":"1999","unstructured":"D. Kramer , \"Api documentation from source code comments: a case study of javadoc,\" in Proceedings of the 17th annual international conference on Computer documentation . ACM , 1999 , pp. 147 -- 153 . D. Kramer, \"Api documentation from source code comments: a case study of javadoc,\" in Proceedings of the 17th annual international conference on Computer documentation. ACM, 1999, pp. 147--153."},{"key":"e_1_3_2_1_4_1","first-page":"26","volume-title":"ACM","author":"Forward A.","year":"2002","unstructured":"A. Forward and T. C. Lethbridge , \" The relevance of software documentation, tools and technologies: a survey,\" in Proceedings of the 2002 ACM symposium on Document engineering . ACM , 2002 , pp. 26 -- 33 . A. Forward and T. C. Lethbridge, \"The relevance of software documentation, tools and technologies: a survey,\" in Proceedings of the 2002 ACM symposium on Document engineering. ACM, 2002, pp. 26--33."},{"key":"e_1_3_2_1_5_1","first-page":"35","volume-title":"IEEE","author":"Haiduc S.","year":"2010","unstructured":"S. Haiduc , J. Aponte , L. Moreno , and A. Marcus , \" On the use of automated text summarization techniques for summarizing source code,\" in 2010 17th Working Conference on Reverse Engineering . IEEE , 2010 , pp. 35 -- 44 . S. Haiduc, J. Aponte, L. Moreno, and A. Marcus, \"On the use of automated text summarization techniques for summarizing source code,\" in 2010 17th Working Conference on Reverse Engineering. IEEE, 2010, pp. 35--44."},{"key":"e_1_3_2_1_6_1","first-page":"3931","volume-title":"Human Language Technologies","author":"LeClair A.","year":"2019","unstructured":"A. LeClair and C. McMillan , \"Recommendations for datasets for source code summarization,\" in Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics : Human Language Technologies , Volume 1 ( Long and Short Papers) , 2019 , pp. 3931 -- 3937 . A. LeClair and C. McMillan, \"Recommendations for datasets for source code summarization,\" in Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), 2019, pp. 3931--3937."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"D. Roy S. Fakhoury and V. Arnaoudova \"Reassessing automatic evaluation metrics for code summarization tasks \" in Proceedings of the ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC\/FSE) 2021.  D. Roy S. Fakhoury and V. Arnaoudova \"Reassessing automatic evaluation metrics for code summarization tasks \" in Proceedings of the ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC\/FSE) 2021.","DOI":"10.1145\/3468264.3468588"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_9_1","volume-title":"A package for automatic evaluation of summaries,\" Text Summarization Branches Out","author":"Lin C.-Y.","year":"2004","unstructured":"C.-Y. Lin , \"Rouge : A package for automatic evaluation of summaries,\" Text Summarization Branches Out , 2004 . C.-Y. Lin, \"Rouge: A package for automatic evaluation of summaries,\" Text Summarization Branches Out, 2004."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00322"},{"key":"e_1_3_2_1_11_1","first-page":"2","volume-title":"A human study of comprehension and code summarization,\" in Proceedings of the 28th International Conference on Program Comprehension","author":"Stapleton S.","year":"2020","unstructured":"S. Stapleton , Y. Gambhir , A. LeClair , Z. Eberhart , W. Weimer , K. Leach , and Y. Huang , \" A human study of comprehension and code summarization,\" in Proceedings of the 28th International Conference on Program Comprehension , 2020 , pp. 2 -- 13 . S. Stapleton, Y. Gambhir, A. LeClair, Z. Eberhart, W. Weimer, K. Leach, and Y. Huang, \"A human study of comprehension and code summarization,\" in Proceedings of the 28th International Conference on Program Comprehension, 2020, pp. 2--13."},{"key":"e_1_3_2_1_12_1","first-page":"4344","volume-title":"Beyond bleu: Training neural machine translation with semantic similarity,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Wieting J.","year":"2019","unstructured":"J. Wieting , T. Berg-Kirkpatrick , K. Gimpel , and G. Neubig , \" Beyond bleu: Training neural machine translation with semantic similarity,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics , 2019 , pp. 4344 -- 4355 . J. Wieting, T. Berg-Kirkpatrick, K. Gimpel, and G. Neubig, \"Beyond bleu: Training neural machine translation with semantic similarity,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, 2019, pp. 4344--4355."},{"key":"e_1_3_2_1_13_1","first-page":"746","volume-title":"Code to comment \u00e2\u0102IJtranslation\u00e2\u0102\u0130 Data, metrics, baselining evaluation,\" in 2020 35th IEEE\/ACM International Conference on Automated Software Engineering (ASE)","author":"Gros D.","year":"2020","unstructured":"D. Gros , H. Sezhiyan , P. Devanbu , and Z. Yu , \" Code to comment \u00e2\u0102IJtranslation\u00e2\u0102\u0130 Data, metrics, baselining evaluation,\" in 2020 35th IEEE\/ACM International Conference on Automated Software Engineering (ASE) , 2020 , pp. 746 -- 757 . D. Gros, H. Sezhiyan, P. Devanbu, and Z. Yu, \"Code to comment \u00e2\u0102IJtranslation\u00e2\u0102\u0130 Data, metrics, baselining evaluation,\" in 2020 35th IEEE\/ACM International Conference on Automated Software Engineering (ASE), 2020, pp. 746--757."},{"key":"e_1_3_2_1_14_1","volume-title":"Sentence-bert: Sentence embeddings using siamese bert-networks,\" in Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing","author":"Reimers N.","year":"2019","unstructured":"N. Reimers and I. Gurevych , \" Sentence-bert: Sentence embeddings using siamese bert-networks,\" in Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing . Association for Computational Linguistics , 2019 . N. Reimers and I. Gurevych, \"Sentence-bert: Sentence embeddings using siamese bert-networks,\" in Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 2019."},{"key":"e_1_3_2_1_15_1","first-page":"101","volume-title":"ACM","author":"Sridhara G.","year":"2011","unstructured":"G. Sridhara , L. Pollock , and K. Vijay-Shanker , \" Automatically detecting and describing high level actions within methods,\" in Proceedings of the 33rd International Conference on Software Engineering . ACM , 2011 , pp. 101 -- 110 . G. Sridhara, L. Pollock, and K. Vijay-Shanker, \"Automatically detecting and describing high level actions within methods,\" in Proceedings of the 33rd International Conference on Software Engineering. ACM, 2011, pp. 101--110."},{"key":"e_1_3_2_1_16_1","first-page":"71","volume-title":"IEEE","author":"Sridhara G.","year":"2011","unstructured":"G. Sridhara , L. Pollock , and K. Vijay-Shanker , \" Generating parameter comments and integrating with method summaries,\" in 2011 IEEE 19th International Conference on Program Comprehension . IEEE , 2011 , pp. 71 -- 80 . G. Sridhara, L. Pollock, and K. Vijay-Shanker, \"Generating parameter comments and integrating with method summaries,\" in 2011 IEEE 19th International Conference on Program Comprehension. IEEE, 2011, pp. 71--80."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2465386"},{"key":"e_1_3_2_1_18_1","first-page":"484","volume-title":"ACM","author":"Moreno L.","year":"2014","unstructured":"L. Moreno , G. Bavota , M. Di Penta , R. Oliveto , A. Marcus , and G. Canfora , \" Automatic generation of release notes,\" in Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering . ACM , 2014 , pp. 484 -- 495 . L. Moreno, G. Bavota, M. Di Penta, R. Oliveto, A. Marcus, and G. Canfora, \"Automatic generation of release notes,\" in Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering. ACM, 2014, pp. 484--495."},{"key":"e_1_3_2_1_19_1","first-page":"193","volume-title":"IEEE","author":"De Lucia A.","year":"2012","unstructured":"A. De Lucia , M. Di Penta , R. Oliveto , A. Panichella , and S. Panichella , \" Using ir methods for labeling source code artifacts: Is it worthwhile?\" in 2012 20th IEEE International Conference on Program Comprehension (ICPC) . IEEE , 2012 , pp. 193 -- 202 . A. De Lucia, M. Di Penta, R. Oliveto, A. Panichella, and S. Panichella, \"Using ir methods for labeling source code artifacts: Is it worthwhile?\" in 2012 20th IEEE International Conference on Program Comprehension (ICPC). IEEE, 2012, pp. 193--202."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2013.2297712"},{"key":"e_1_3_2_1_21_1","first-page":"390","volume-title":"ACM","author":"Rodeghero P.","year":"2014","unstructured":"P. Rodeghero , C. McMillan , P. W. McBurney , N. Bosch , and S. D'Mello , \"Improving automated source code summarization via an eye-tracking study of programmers,\" in Proceedings of the 36th international conference on Software engineering . ACM , 2014 , pp. 390 -- 401 . P. Rodeghero, C. McMillan, P. W. McBurney, N. Bosch, and S. D'Mello, \"Improving automated source code summarization via an eye-tracking study of programmers,\" in Proceedings of the 36th international conference on Software engineering. ACM, 2014, pp. 390--401."},{"key":"e_1_3_2_1_22_1","first-page":"2073","article-title":"Summarizing source code using a neural attention model,\" in Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1","author":"Iyer S.","year":"2016","unstructured":"S. Iyer , I. Konstas , A. Cheung , and L. Zettlemoyer , \" Summarizing source code using a neural attention model,\" in Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1 : Long Papers) , 2016 , pp. 2073 -- 2083 . S. Iyer, I. Konstas, A. Cheung, and L. Zettlemoyer, \"Summarizing source code using a neural attention model,\" in Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), 2016, pp. 2073--2083.","journal-title":"Long Papers)"},{"key":"e_1_3_2_1_23_1","first-page":"135","volume-title":"IEEE Press","author":"Jiang S.","year":"2017","unstructured":"S. Jiang , A. Armaly , and C. McMillan , \" Automatically generating commit messages from diffs using neural machine translation,\" in Proceedings of the 32nd IEEE\/ACM International Conference on Automated Software Engineering . IEEE Press , 2017 , pp. 135 -- 146 . S. Jiang, A. Armaly, and C. McMillan, \"Automatically generating commit messages from diffs using neural machine translation,\" in Proceedings of the 32nd IEEE\/ACM International Conference on Automated Software Engineering. IEEE Press, 2017, pp. 135--146."},{"key":"e_1_3_2_1_24_1","first-page":"200","volume-title":"ACM","author":"Hu X.","year":"2018","unstructured":"X. Hu , G. Li , X. Xia , D. Lo , and Z. Jin , \" Deep code comment generation,\" in Proceedings of the 26th Conference on Program Comprehension . ACM , 2018 , pp. 200 -- 210 . X. Hu, G. Li, X. Xia, D. Lo, and Z. Jin, \"Deep code comment generation,\" in Proceedings of the 26th Conference on Program Comprehension. ACM, 2018, pp. 200--210."},{"key":"e_1_3_2_1_25_1","volume-title":"code2seq: Generating sequences from structured representations of code,\" International Conference on Learning Representations","author":"Alon U.","year":"2019","unstructured":"U. Alon , S. Brody , O. Levy , and E. Yahav , \" code2seq: Generating sequences from structured representations of code,\" International Conference on Learning Representations , 2019 . U. Alon, S. Brody, O. Levy, and E. Yahav, \"code2seq: Generating sequences from structured representations of code,\" International Conference on Learning Representations, 2019."},{"key":"e_1_3_2_1_26_1","first-page":"1","article-title":"\"code2vec: Learning distributed representations of code,\" Proceedings of the ACM on Programming Languages, vol. 3, no","author":"Alon U.","year":"2019","unstructured":"U. Alon , M. Zilberstein , O. Levy , and E. Yahav , \"code2vec: Learning distributed representations of code,\" Proceedings of the ACM on Programming Languages, vol. 3, no . POPL , pp. 1 -- 29 , 2019 . U. Alon, M. Zilberstein, O. Levy, and E. Yahav, \"code2vec: Learning distributed representations of code,\" Proceedings of the ACM on Programming Languages, vol. 3, no. POPL, pp. 1--29, 2019.","journal-title":"POPL"},{"key":"e_1_3_2_1_27_1","first-page":"795","volume-title":"IEEE Press","author":"LeClair A.","year":"2019","unstructured":"A. LeClair , S. Jiang , and C. McMillan , \" A neural model for generating natural language summaries of program subroutines,\" in Proceedings of the 41st International Conference on Software Engineering . IEEE Press , 2019 , pp. 795 -- 806 . A. LeClair, S. Jiang, and C. McMillan, \"A neural model for generating natural language summaries of program subroutines,\" in Proceedings of the 41st International Conference on Software Engineering. IEEE Press, 2019, pp. 795--806."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"A. LeClair S. Haque L. Wu and C. McMillan \"Improved code summarization via a graph neural network \" in 28th ACM\/IEEE International Conference on Program Comprehension (ICPC'20) 2020.  A. LeClair S. Haque L. Wu and C. McMillan \"Improved code summarization via a graph neural network \" in 28th ACM\/IEEE International Conference on Program Comprehension (ICPC'20) 2020.","DOI":"10.1145\/3387904.3389268"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"W. U. Ahmad S. Chakraborty B. Ray and K.-W. Chang \"A transformer-based approach for source code summarization \" arXiv preprint arXiv:2005.00653 2020.  W. U. Ahmad S. Chakraborty B. Ray and K.-W. Chang \"A transformer-based approach for source code summarization \" arXiv preprint arXiv:2005.00653 2020.","DOI":"10.18653\/v1\/2020.acl-main.449"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"S. Haque A. LeClair L. Wu and C. McMillan \"Improved automatic summarization of subroutines via attention to file context \" International Conference on Mining Software Repositories 2020.  S. Haque A. LeClair L. Wu and C. McMillan \"Improved automatic summarization of subroutines via attention to file context \" International Conference on Mining Software Repositories 2020.","DOI":"10.1145\/3379597.3387449"},{"key":"e_1_3_2_1_31_1","unstructured":"R. H. C. T. L. Chin Yee Lee Hengfeng Li. (2019) Nltk translate bleu score calculator v3.3. [Online]. Available: https:\/\/www.nltk.org\/modules\/nltk\/translate\/bleuscore.html  R. H. C. T. L. Chin Yee Lee Hengfeng Li. (2019) Nltk translate bleu score calculator v3.3. [Online]. Available: https:\/\/www.nltk.org\/modules\/nltk\/translate\/bleuscore.html"},{"key":"e_1_3_2_1_32_1","first-page":"2241 17","volume-title":"Denmark: Association for Computational Linguistics","author":"Novikova J.","year":"2017","unstructured":"J. Novikova , O. Du\u0161ek , A. Cercas Curry , and V. Rieser , \" Why we need new evaluation metrics for NLG,\" in Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing. Copenhagen , Denmark: Association for Computational Linguistics , Sep. 2017 , pp. 2241 -- 2252 . [Online]. Available : https:\/\/aclanthology.org\/D 17 - 1238 J. Novikova, O. Du\u0161ek, A. Cercas Curry, and V. Rieser, \"Why we need new evaluation metrics for NLG,\" in Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing. Copenhagen, Denmark: Association for Computational Linguistics, Sep. 2017, pp. 2241--2252. [Online]. Available: https:\/\/aclanthology.org\/D17-1238"},{"key":"e_1_3_2_1_33_1","first-page":"355","volume-title":"Best practices for the human evaluation of automatically generated text,\" in Proceedings of the 12th International Conference on Natural Language Generation","author":"Van Der Lee C.","year":"2019","unstructured":"C. Van Der Lee , A. Gatt , E. Van Miltenburg , S. Wubben , and E. Krahmer , \" Best practices for the human evaluation of automatically generated text,\" in Proceedings of the 12th International Conference on Natural Language Generation , 2019 , pp. 355 -- 368 . C. Van Der Lee, A. Gatt, E. Van Miltenburg, S. Wubben, and E. Krahmer, \"Best practices for the human evaluation of automatically generated text,\" in Proceedings of the 12th International Conference on Natural Language Generation, 2019, pp. 355--368."},{"key":"e_1_3_2_1_34_1","first-page":"2748","volume-title":"Sentence mover\u00e2\u0102Źs similarity: Automatic evaluation for multi-sentence texts,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Clark E.","year":"2019","unstructured":"E. Clark , A. Celikyilmaz , and N. A. Smith , \" Sentence mover\u00e2\u0102Źs similarity: Automatic evaluation for multi-sentence texts,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics , 2019 , pp. 2748 -- 2760 . E. Clark, A. Celikyilmaz, and N. A. Smith, \"Sentence mover\u00e2\u0102Źs similarity: Automatic evaluation for multi-sentence texts,\" in Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, 2019, pp. 2748--2760."},{"key":"e_1_3_2_1_35_1","unstructured":"S. Banerjee and A. Lavie \"Meteor: An automatic metric for mt evaluation with improved correlation with human judgments \" in Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization 2005 pp. 65--72.  S. Banerjee and A. Lavie \"Meteor: An automatic metric for mt evaluation with improved correlation with human judgments \" in Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization 2005 pp. 65--72."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"M. Allamanis \"The adverse effects of code duplication in machine learning models of code \" in Proceedings of the 2019 ACM SIGPLAN International Symposium on New Ideas New Paradigms and Reflections on Programming and Software 2019 pp. 143--153.  M. Allamanis \"The adverse effects of code duplication in machine learning models of code \" in Proceedings of the 2019 ACM SIGPLAN International Symposium on New Ideas New Paradigms and Reflections on Programming and Software 2019 pp. 143--153.","DOI":"10.1145\/3359591.3359735"},{"key":"e_1_3_2_1_37_1","first-page":"43","volume-title":"ACM","author":"Sridhara G.","year":"2010","unstructured":"G. Sridhara , E. Hill , D. Muppaneni , L. Pollock , and K. Vijay-Shanker , \" Towards automatically generating summary comments for java methods,\" in Proceedings of the IEEE\/ACM international conference on Automated software engineering . ACM , 2010 , pp. 43 -- 52 . G. Sridhara, E. Hill, D. Muppaneni, L. Pollock, and K. Vijay-Shanker, \"Towards automatically generating summary comments for java methods,\" in Proceedings of the IEEE\/ACM international conference on Automated software engineering. ACM, 2010, pp. 43--52."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1017\/S1930297500002205"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Z. Eberhart A. LeClair and C. McMillan \"Automatically extracting subroutine summary descriptions from unstructured comments \" arXiv preprint arXiv:1912.10198 2019.  Z. Eberhart A. LeClair and C. McMillan \"Automatically extracting subroutine summary descriptions from unstructured comments \" arXiv preprint arXiv:1912.10198 2019.","DOI":"10.1109\/SANER48275.2020.9054789"},{"key":"e_1_3_2_1_40_1","volume-title":"Evolution and Reengineering","author":"Haque S.","year":"2021","unstructured":"S. Haque , A. Bansal , L. Wu , and C. McMillan , \" Action word prediction for neural source code summarization,\" 28th IEEE International Conference on Software Analysis , Evolution and Reengineering , 2021 . S. Haque, A. Bansal, L. Wu, and C. McMillan, \"Action word prediction for neural source code summarization,\" 28th IEEE International Conference on Software Analysis, Evolution and Reengineering, 2021."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"A. Wood P. Rodeghero A. Armaly and C. McMillan \"Detecting speech act types in developer question\/answer conversations during bug repair \" in Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. ACM 2018 pp. 491--502.  A. Wood P. Rodeghero A. Armaly and C. McMillan \"Detecting speech act types in developer question\/answer conversations during bug repair \" in Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. ACM 2018 pp. 491--502.","DOI":"10.1145\/3236024.3236031"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1080\/19312458.2011.568376"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1177\/1049732305276687"},{"key":"e_1_3_2_1_44_1","first-page":"1","article-title":"Kappa coefficient","author":"Kraemer H. C.","year":"2014","unstructured":"H. C. Kraemer , \" Kappa coefficient ,\" Wiley StatsRef: Statistics Reference Online , pp. 1 -- 4 , 2014 . H. C. Kraemer, \"Kappa coefficient,\" Wiley StatsRef: Statistics Reference Online, pp. 1--4, 2014.","journal-title":"Wiley StatsRef: Statistics Reference Online"},{"issue":"3","key":"e_1_3_2_1_45_1","first-page":"411","article-title":"Reliability in content analysis: Some common misconceptions and recommendations","volume":"30","author":"Krippendorff K.","year":"2004","unstructured":"K. Krippendorff , \" Reliability in content analysis: Some common misconceptions and recommendations ,\" Human communication research , vol. 30 , no. 3 , pp. 411 -- 433 , 2004 . K. Krippendorff, \"Reliability in content analysis: Some common misconceptions and recommendations,\" Human communication research, vol. 30, no. 3, pp. 411--433, 2004.","journal-title":"Human communication research"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1162\/089120105774321109"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10742-011-0077-3"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1080\/00909889909365539"},{"key":"e_1_3_2_1_49_1","article-title":"A wizard of oz study simulating api usage dialogues with a virtual assistant","author":"Eberhart Z.","year":"2020","unstructured":"Z. Eberhart , A. Bansal , and C. Mcmillan , \" A wizard of oz study simulating api usage dialogues with a virtual assistant ,\" IEEE Transactions on Software Engineering , 2020 . Z. Eberhart, A. Bansal, and C. Mcmillan, \"A wizard of oz study simulating api usage dialogues with a virtual assistant,\" IEEE Transactions on Software Engineering, 2020.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_50_1","volume-title":"Bertscore: Evaluating text generation with bert,\" arXiv preprint arXiv:1904.09675","author":"Zhang T.","year":"2019","unstructured":"T. Zhang , V. Kishore , F. Wu , K. Q. Weinberger , and Y. Artzi , \" Bertscore: Evaluating text generation with bert,\" arXiv preprint arXiv:1904.09675 , 2019 . T. Zhang, V. Kishore, F. Wu, K. Q. Weinberger, and Y. Artzi, \"Bertscore: Evaluating text generation with bert,\" arXiv preprint arXiv:1904.09675, 2019."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1469-8137.1912.tb05611.x"},{"key":"e_1_3_2_1_52_1","unstructured":"T. Tanimoto \"An elementary mathematical theory of classification and prediction \" IBM Internal Report 1958.  T. Tanimoto \"An elementary mathematical theory of classification and prediction \" IBM Internal Report 1958."},{"key":"e_1_3_2_1_53_1","first-page":"221","volume-title":"IEEE","author":"Croft D.","year":"2013","unstructured":"D. Croft , S. Coupland , J. Shell , and S. Brown , \" A fast and efficient semantic short text similarity metric,\" in 2013 13th UK workshop on computational intelligence (UKCI) . IEEE , 2013 , pp. 221 -- 227 . D. Croft, S. Coupland, J. Shell, and S. Brown, \"A fast and efficient semantic short text similarity metric,\" in 2013 13th UK workshop on computational intelligence (UKCI). IEEE, 2013, pp. 221--227."},{"key":"e_1_3_2_1_54_1","volume-title":"Sentence similarity based on semantic nets and corpus statistics,\" IEEE transactions on knowledge and data engineering","author":"Li Y.","unstructured":"Y. Li , D. McLean , Z. A. Bandar , J. D. O'shea , and K. Crockett , \" Sentence similarity based on semantic nets and corpus statistics,\" IEEE transactions on knowledge and data engineering , vol. 18 , no. 8, pp. 1138--1150, 2006. Y. Li, D. McLean, Z. A. Bandar, J. D. O'shea, and K. Crockett, \"Sentence similarity based on semantic nets and corpus statistics,\" IEEE transactions on knowledge and data engineering, vol. 18, no. 8, pp. 1138--1150, 2006."},{"key":"e_1_3_2_1_55_1","first-page":"231","volume-title":"computer applications","author":"Fellbaum C.","year":"2010","unstructured":"C. Fellbaum , \"Wordnet,\" in Theory and applications of ontology : computer applications . Springer , 2010 , pp. 231 -- 243 . C. Fellbaum, \"Wordnet,\" in Theory and applications of ontology: computer applications. Springer, 2010, pp. 231--243."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"crossref","unstructured":"T. Kenter and M. De Rijke \"Short text similarity with word embeddings \" in Proceedings of the 24th ACM international on conference on information and knowledge management 2015 pp. 1411--1420.  T. Kenter and M. De Rijke \"Short text similarity with word embeddings \" in Proceedings of the 24th ACM international on conference on information and knowledge management 2015 pp. 1411--1420.","DOI":"10.1145\/2806416.2806475"},{"key":"e_1_3_2_1_57_1","volume-title":"Efficient estimation of word representations in vector space,\" arXiv preprint arXiv:1301.3781","author":"Mikolov T.","year":"2013","unstructured":"T. Mikolov , K. Chen , G. Corrado , and J. Dean , \" Efficient estimation of word representations in vector space,\" arXiv preprint arXiv:1301.3781 , 2013 . T. Mikolov, K. Chen, G. Corrado, and J. Dean, \"Efficient estimation of word representations in vector space,\" arXiv preprint arXiv:1301.3781, 2013."},{"key":"e_1_3_2_1_58_1","first-page":"1532","volume-title":"GloVe: Global vectors for word representation,\" in Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Pennington J.","year":"2014","unstructured":"J. Pennington , R. Socher , and C. Manning , \" GloVe: Global vectors for word representation,\" in Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP) . Association for Computational Linguistics , 2014 , pp. 1532 -- 1543 . J. Pennington, R. Socher, and C. Manning, \"GloVe: Global vectors for word representation,\" in Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics, 2014, pp. 1532--1543."},{"key":"e_1_3_2_1_59_1","volume-title":"Distributed representations of sentences and documents,\" in Proceedings of the 31st International Conference on International Conference on Machine Learning -","author":"Le Q.","year":"2014","unstructured":"Q. Le and T. Mikolov , \" Distributed representations of sentences and documents,\" in Proceedings of the 31st International Conference on International Conference on Machine Learning - Volume 32 . JMLR.org, 2014 , p. II\u00e2\u0102\u015e 1188\u00e2\u0102\u015eII\u00e2\u0102\u015e1196. Q. Le and T. Mikolov, \"Distributed representations of sentences and documents,\" in Proceedings of the 31st International Conference on International Conference on Machine Learning - Volume 32. JMLR.org, 2014, p. II\u00e2\u0102\u015e1188\u00e2\u0102\u015eII\u00e2\u0102\u015e1196."},{"key":"e_1_3_2_1_60_1","volume-title":"Supervised learning of universal sentence representations from natural language inference data","author":"Conneau A.","year":"2018","unstructured":"A. Conneau , D. Kiela , H. Schwenk , L. Barrault , and A. Bordes , \" Supervised learning of universal sentence representations from natural language inference data ,\" 2018 . A. Conneau, D. Kiela, H. Schwenk, L. Barrault, and A. Bordes, \"Supervised learning of universal sentence representations from natural language inference data,\" 2018."},{"key":"e_1_3_2_1_61_1","first-page":"4171","volume-title":"BERT: Pre-training of deep bidirectional transformers for language understanding,\" in Proceedings of the 2019 Conference of the North American","author":"Devlin J.","year":"2019","unstructured":"J. Devlin , M.-W. Chang , K. Lee , and K. Toutanova , \" BERT: Pre-training of deep bidirectional transformers for language understanding,\" in Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). Association for Computational Linguistics , 2019 , pp. 4171 -- 4186 . J. Devlin, M.-W. Chang, K. Lee, and K. Toutanova, \"BERT: Pre-training of deep bidirectional transformers for language understanding,\" in Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). Association for Computational Linguistics, 2019, pp. 4171--4186."},{"key":"e_1_3_2_1_62_1","volume-title":"Roberta: A robustly optimized bert pretraining approach","author":"Liu Y.","year":"2019","unstructured":"Y. Liu , M. Ott , N. Goyal , J. Du , M. Joshi , D. Chen , O. Levy , M. Lewis , L. Zettlemoyer , and V. Stoyanov , \" Roberta: A robustly optimized bert pretraining approach ,\" 2019 . Y. Liu, M. Ott, N. Goyal, J. Du, M. Joshi, D. Chen, O. Levy, M. Lewis, L. Zettlemoyer, and V. Stoyanov, \"Roberta: A robustly optimized bert pretraining approach,\" 2019."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(88)90021-0"},{"key":"e_1_3_2_1_64_1","unstructured":"J. E. Ramos \"Using tf-idf to determine word relevance in document queries \" 2003.  J. E. Ramos \"Using tf-idf to determine word relevance in document queries \" 2003."},{"key":"e_1_3_2_1_65_1","volume-title":"N. Hua, N. Limtiaco, R. S. John, N. Constant, M. Guajardo-Cespedes, S. Yuan, C. Tar, Y.-H. Sung, B. Strope, and R. Kurzweil, \"Universal sentence encoder,\"","author":"Cer D.","year":"2018","unstructured":"D. Cer , Y. Yang , S. yi Kong , N. Hua, N. Limtiaco, R. S. John, N. Constant, M. Guajardo-Cespedes, S. Yuan, C. Tar, Y.-H. Sung, B. Strope, and R. Kurzweil, \"Universal sentence encoder,\" 2018 . D. Cer, Y. Yang, S. yi Kong, N. Hua, N. Limtiaco, R. S. John, N. Constant, M. Guajardo-Cespedes, S. Yuan, C. Tar, Y.-H. Sung, B. Strope, and R. Kurzweil, \"Universal sentence encoder,\" 2018."},{"key":"e_1_3_2_1_66_1","first-page":"5998","volume-title":"Attention is all you need,\" in Advances in neural information processing systems","author":"Vaswani A.","year":"2017","unstructured":"A. Vaswani , N. Shazeer , N. Parmar , J. Uszkoreit , L. Jones , A. N. Gomez , \u0141. Kaiser, and I. Polosukhin , \" Attention is all you need,\" in Advances in neural information processing systems , 2017 , pp. 5998 -- 6008 . A. Vaswani, N. Shazeer, N. Parmar, J. Uszkoreit, L. Jones, A. N. Gomez, \u0141. Kaiser, and I. Polosukhin, \"Attention is all you need,\" in Advances in neural information processing systems, 2017, pp. 5998--6008."},{"key":"e_1_3_2_1_67_1","volume-title":"Effective approaches to attention-based neural machine translation,\" arXiv preprint arXiv:1508.04025","author":"Luong M.-T.","year":"2015","unstructured":"M.-T. Luong , H. Pham , and C. D. Manning , \" Effective approaches to attention-based neural machine translation,\" arXiv preprint arXiv:1508.04025 , 2015 . M.-T. Luong, H. Pham, and C. D. Manning, \"Effective approaches to attention-based neural machine translation,\" arXiv preprint arXiv:1508.04025, 2015."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1179"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1152\/advan.00007.2009"},{"key":"e_1_3_2_1_70_1","volume-title":"Statistics without maths for psychology. Pearson education","author":"Dancey C. P.","year":"2007","unstructured":"C. P. Dancey and J. Reidy , Statistics without maths for psychology. Pearson education , 2007 . C. P. Dancey and J. Reidy, Statistics without maths for psychology. Pearson education, 2007."}],"event":{"name":"ICPC '22: 30th International Conference on Program Comprehension","location":"Virtual Event","acronym":"ICPC '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 30th IEEE\/ACM International Conference on Program Comprehension"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524610.3527909","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3524610.3527909","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3524610.3527909","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:53Z","timestamp":1750183793000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524610.3527909"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,16]]},"references-count":70,"alternative-id":["10.1145\/3524610.3527909","10.1145\/3524610"],"URL":"https:\/\/doi.org\/10.1145\/3524610.3527909","relation":{},"subject":[],"published":{"date-parts":[[2022,5,16]]},"assertion":[{"value":"2022-10-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}