{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T22:56:58Z","timestamp":1773442618355,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":82,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62072227, 62202219"],"award-info":[{"award-number":["62072227, 62202219"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2019YFE0105500"],"award-info":[{"award-number":["2019YFE0105500"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Key Research and Development Program of Jiangsu","award":["BE2021002-2"],"award-info":[{"award-number":["BE2021002-2"]}]},{"name":"Swiss National Science Foundation","award":["200021_197227"],"award-info":[{"award-number":["200021_197227"]}]},{"name":"Research Council of Norway","award":["309494"],"award-info":[{"award-number":["309494"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616245","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"275-287","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["EvaCRC: Evaluating Code Review Comments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0406-2263","authenticated-orcid":false,"given":"Lanxin","family":"Yang","sequence":"first","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-5157-1118","authenticated-orcid":false,"given":"Jinwei","family":"Xu","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5051-6967","authenticated-orcid":false,"given":"Yifan","family":"Zhang","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9159-5331","authenticated-orcid":false,"given":"He","family":"Zhang","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0193-6823","authenticated-orcid":false,"given":"Alberto","family":"Bacchelli","sequence":"additional","affiliation":[{"name":"University of Zurich, Zurich, Switzerland"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/3155562.3155579"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606617"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2950290.2950323"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-015-9366-8"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2576451"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.21"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3511560"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1010933404324"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1188\/14.ONF.545-547"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387904.3389270"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER50967.2021.00060"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.2307\/1176100"},{"key":"e_1_3_2_2_13_1","unstructured":"John W Creswell and J David Creswell. 2017. Research design: Qualitative quantitative and mixed methods approaches. SAGE."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2011.36"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.131"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.110951"},{"key":"e_1_3_2_2_17_1","first-page":"11","article-title":"Deep learning-based semantic feature extraction: A literature review and future directions","volume":"21","author":"Deng Letian","year":"2023","unstructured":"Letian Deng and Yanru Zhao. 2023. Deep learning-based semantic feature extraction: A literature review and future directions. ZTE Communications, 21, 2 (2023), 11\u201317.","journal-title":"ZTE Communications"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/1558689812437186"},{"key":"e_1_3_2_2_19_1","volume-title":"The research act: A theoretical introduction to sociological methods","author":"Denzin Norman K","unstructured":"Norman K Denzin. 2017. The research act: A theoretical introduction to sociological methods. Routledge."},{"key":"e_1_3_2_2_20_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. ACL, 29\u201335","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of deep bidirectional transformers for language understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. ACL, 29\u201335."},{"key":"e_1_3_2_2_21_1","volume-title":"Towards a taxonomy of code review smells. Information and Software Technology, 142","author":"Do\u011fan Emre","year":"2022","unstructured":"Emre Do\u011fan and Eray T\u00fcz\u00fcn. 2022. Towards a taxonomy of code review smells. Information and Software Technology, 142 (2022), 106737:1\u201324."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09909-5"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183399.3183411"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380414"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2019.06.005"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1147\/sj.153.0182"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_2_28_1","unstructured":"Gensim. 2023. Word2vec embeddings. https:\/\/radimrehurek.com\/gensim\/models\/word2vec.html"},{"key":"e_1_3_2_2_29_1","unstructured":"GitLab. 2020. Code review guidelines. https:\/\/docs.gitlab.com\/ee\/development\/code_review.html (accessed 01.25.2023)"},{"key":"e_1_3_2_2_30_1","unstructured":"Google. 2019. Google\u2019s engineering practices documentation. https:\/\/google.github.io\/eng-practices\/review\/ (accessed 01.25.2023)"},{"key":"e_1_3_2_2_31_1","volume-title":"Concerns identified in code review: A fine-grained, faceted classification. Information and Software Technology, 153","author":"Gunawardena Sanuri","year":"2023","unstructured":"Sanuri Gunawardena, Ewan Tempero, and Kelly Blincoe. 2023. Concerns identified in code review: A fine-grained, faceted classification. Information and Software Technology, 153 (2023), 107054:1\u201314."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.740"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/SCAM51674.2020.00010"},{"key":"e_1_3_2_2_34_1","volume-title":"AJM Rahman, and Amiangshu Bosu.","author":"Hasan Masum","year":"2021","unstructured":"Masum Hasan, Anindya Iqbal, Mohammad Rafid Ul Islam, AJM Rahman, and Amiangshu Bosu. 2021. Using a balanced scorecard to identify opportunities to improve code review effectiveness: An industrial experience report. Empirical Software Engineering, 26, 6 (2021), 129:1\u201334."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549119"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-021-10035-z"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.2307\/2392366"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1052"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-016-9493-x"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1181"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.2015.7332457"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09837-4"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v29i1.9513"},{"key":"e_1_3_2_2_44_1","volume-title":"Deep learning. Nature, 521, 7553","author":"LeCun Yann","year":"2015","unstructured":"Yann LeCun, Yoshua Bengio, and Geoffrey Hinton. 2015. Deep learning. Nature, 521, 7553 (2015), 436\u2013444."},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.18293\/SEKE2017-039"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1951.10500769"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-015-9381-9"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-62105-0_5"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.35"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2016.65"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2912113"},{"key":"e_1_3_2_2_52_1","first-page":"1","article-title":"Modeling and mitigating human annotation errors to design efficient stream processing systems with human-in-the-loop machine learning","volume":"102772","author":"Pandey Rahul","year":"2022","unstructured":"Rahul Pandey, Hemant Purohit, Carlos Castillo, and Valerie L Shalin. 2022. Modeling and mitigating human annotation errors to design efficient stream processing systems with human-in-the-loop machine learning. International Journal of Human-Computer Studies, 102772:1\u201312.","journal-title":"International Journal of Human-Computer Studies"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274404"},{"key":"e_1_3_2_2_54_1","volume-title":"Proceedings of the 33rd International Conference on Neural Information Processing Systems. Curran, 1396\u20131408","author":"Rabanser Stephan","year":"2019","unstructured":"Stephan Rabanser, Stephan G\u00fcnnemann, and Zachary C Lipton. 2019. Failing loudly: An empirical study of methods for detecting dataset shift. In Proceedings of the 33rd International Conference on Neural Information Processing Systems. Curran, 1396\u20131408."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1079\/PNS2004399"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.18293\/SEKE2018-097"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2017.17"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-011-5256-5"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/2491411.2491444"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183519.3183525"},{"key":"e_1_3_2_2_61_1","volume-title":"Guide to advanced empirical software engineering","author":"Shull Forrest","unstructured":"Forrest Shull, Janice Singer, and Dag IK Sj\u00f8berg. 2007. Guide to advanced empirical software engineering. Springer."},{"key":"e_1_3_2_2_62_1","unstructured":"IEEE Computer Society. 2008. IEEE 1028-2008 - IEEE standard for software reviews and audits. https:\/\/standards.ieee.org\/standard\/1028-2008.html (accessed 12.15.2022)"},{"key":"e_1_3_2_2_63_1","volume-title":"Proceedings of the 42nd IEEE\/ACM International Conference on Software Engineering. ACM, 1171\u20131182","author":"Spadini Davide","year":"2020","unstructured":"Davide Spadini, G\u00fcl \u00c7alikli, and Alberto Bacchelli. 2020. Primers or reminders? The effects of existing review comments on code review. In Proceedings of the 42nd IEEE\/ACM International Conference on Software Engineering. ACM, 1171\u20131182."},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00110"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1177\/1098214005283748"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1547-5069.2001.00253.x"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-015-0465-6"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-74958-5_38"},{"key":"e_1_3_2_2_69_1","volume-title":"Proceedings of the 31st Conference on Neural Information Processing Systems. Curran, 5998\u20136008","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141 ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Proceedings of the 31st Conference on Neural Information Processing Systems. Curran, 5998\u20136008."},{"key":"e_1_3_2_2_70_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning. PMLR, 6555\u20136565","author":"Wang Dilin","year":"2019","unstructured":"Dilin Wang, Chengyue Gong, and Qiang Liu. 2019. Improving neural language modeling via adversarial training. In Proceedings of the 36th International Conference on Machine Learning. PMLR, 6555\u20136565."},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.111009"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/QRS54544.2021.00104"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4615-4625-2"},{"key":"e_1_3_2_2_74_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning. PMLR, 3780\u20133788","author":"Wu Xizhu","year":"2017","unstructured":"Xizhu Wu and Zhihua Zhou. 2017. A unified view of multi-label performance measures. In Proceedings of the 34th International Conference on Machine Learning. PMLR, 3780\u20133788."},{"key":"e_1_3_2_2_75_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction, 6, CSCW1","author":"Gon\u00e7alves Pavl\u00edna Wurzel","year":"2022","unstructured":"Pavl\u00edna Wurzel Gon\u00e7alves, G\u00fcl \u00c7alikli, and Alberto Bacchelli. 2022. Interpersonal conflicts during code review: Developers\u2019 experience and practices. Proceedings of the ACM on Human-Computer Interaction, 6, CSCW1 (2022), 1\u201333."},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","unstructured":"Lanxin Yang Jinwei Xu Yifan Zhang He Zhang and Alberto Bacchelli. 2023. Data and materials. https:\/\/doi.org\/10.5281\/zenodo.8297481 10.5281\/zenodo.8297481","DOI":"10.5281\/zenodo.8297481"},{"key":"e_1_3_2_2_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510457.3513055"},{"key":"e_1_3_2_2_78_1","volume-title":"Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems","author":"Yang Zhilin","year":"2019","unstructured":"Zhilin Yang, Zihang Dai, Yiming Yang, Jaime Carbonell, Russ R Salakhutdinov, and Quoc V Le. 2019. XLNet: Generalized autoregressive pretraining for language understanding. In Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019. Curran, 5754\u20135764."},{"key":"e_1_3_2_2_79_1","first-page":"2","article-title":"A lightweight sentiment analysis method","volume":"17","author":"Yu Qingshuang","year":"2019","unstructured":"Qingshuang Yu, Jie Zhou, and Wenjuan Gong. 2019. A lightweight sentiment analysis method. ZTE Communications, 17, 3 (2019), 2\u20138.","journal-title":"ZTE Communications"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-021-10076-4"},{"key":"e_1_3_2_2_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3239235.3239525"},{"key":"e_1_3_2_2_82_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2013.39"}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616245","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616245","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:03Z","timestamp":1750178163000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616245"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":82,"alternative-id":["10.1145\/3611643.3616245","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616245","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}