{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T07:17:35Z","timestamp":1783408655665,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"CNPq \u2013 National Council for Scientific and Technological Development","award":["Universal 404406\/2023-8"],"award-info":[{"award-number":["Universal 404406\/2023-8"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786181.3788728","type":"proceedings-article","created":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:28:24Z","timestamp":1783405704000},"page":"105-112","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Benchmarking LLM Commit Message Generation through a Developer-centric Pairwise Preference Framework"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-2405-046X","authenticated-orcid":false,"given":"Lucas","family":"Aguiar","sequence":"first","affiliation":[{"name":"State University of Ceara, Fortaleza, Ceara, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6980-5834","authenticated-orcid":false,"given":"Matheus","family":"Freitas","sequence":"additional","affiliation":[{"name":"State University of Ceara, Fortaleza, Ceara, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1775-7259","authenticated-orcid":false,"given":"Matheus","family":"Paixao","sequence":"additional","affiliation":[{"name":"State University of Ceara, Fortaleza, Ceara, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5080-1688","authenticated-orcid":false,"given":"Rafael","family":"Carmo","sequence":"additional","affiliation":[{"name":"Federal University of Ceara, Fortaleza, Ceara, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,6]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","unstructured":"Lucas Almeida\u00a0Aguiar Francisco\u00a0Matheus Fernandes\u00a0Freitas Matheus Paixao and Rafael Augusto Ferreira\u00a0do Carmo. 2026. Scripts Software and Dataset for \"Benchmarking LLM Commit Message Generation through a Developer- centric Pairwise Preference Framework\". 10.5281\/zenodo.18369469","DOI":"10.5281\/zenodo.18369469"},{"key":"e_1_3_3_2_3_2","unstructured":"Elo Arpad. 1978. The rating of chessplayers past and present. Arco Pub 216 (1978)."},{"key":"e_1_3_3_2_4_2","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan et\u00a0al. 2022. Training a helpful and harmless assistant with reinforcement learning from human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2204.05862 (2022)."},{"key":"e_1_3_3_2_5_2","first-page":"65","volume-title":"Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization","author":"Banerjee Satanjeev","year":"2005","unstructured":"Satanjeev Banerjee and Alon Lavie. 2005. METEOR: An automatic metric for MT evaluation with improved correlation with human judgments. In Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization. 65\u201372."},{"key":"e_1_3_3_2_6_2","unstructured":"Meriem Boubdir Edward Kim Beyza Ermis Sara Hooker and Marzieh Fadaee. 2023. Elo uncovered: Robustness and best practices in language model evaluation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2311.17295 (2023)."},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"crossref","unstructured":"Ralph\u00a0Allan Bradley and Milton\u00a0E. Terry. 1952. Rank Analysis of Incomplete Block Designs: I. The Method of Paired Comparisons. Biometrika 39 3\/4 (1952) 324\u2013345. http:\/\/www.jstor.org\/stable\/2334029","DOI":"10.1093\/biomet\/39.3-4.324"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/1858996.1859005"},{"key":"e_1_3_3_2_9_2","unstructured":"Wei-Lin Chiang Lianmin Zheng Ying Sheng Anastasios\u00a0Nikolas Angelopoulos Tianle Li Dacheng Li Hao Zhang Banghua Zhu Michael Jordan Joseph\u00a0E. Gonzalez and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. arxiv:https:\/\/arXiv.org\/abs\/2403.04132\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2403.04132"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510069"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.5555\/2486788.2486844"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416546"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/WCRE.2010.13"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00076"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"crossref","unstructured":"Jiawei Li David Farag\u00f3 Christian Petrov and Iftekhar Ahmed. 2024. Only diff is not enough: Generating commit messages leveraging reasoning and action of large language model. Proceedings of the ACM on Software Engineering 1 FSE (2024) 745\u2013766.","DOI":"10.1145\/3643760"},{"key":"e_1_3_3_2_16_2","first-page":"74","volume-title":"Text summarization branches out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74\u201381."},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"crossref","unstructured":"Shangqing Liu Cuiyun Gao Sen Chen Lun\u00a0Yiu Nie and Yang Liu. 2020. Atom: Commit message generation based on abstract syntax tree and hybrid ranking. IEEE Transactions on Software Engineering 48 5 (2020) 1800\u20131817.","DOI":"10.1109\/TSE.2020.3038681"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238190"},{"key":"e_1_3_3_2_19_2","unstructured":"Pablo Loyola Edison Marrese-Taylor and Yutaka Matsuo. 2017. A neural architecture for generating natural language descriptions from source code changes. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1704.04856 (2017)."},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2010.5463344"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/2597008.2597149"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.2000.883028"},{"key":"e_1_3_3_2_23_2","first-page":"311","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311\u2013318."},{"key":"e_1_3_3_2_24_2","unstructured":"Weisong Sun Yun Miao Yuekang Li Hongyu Zhang Chunrong Fang Yi Liu Gelei Deng Yang Liu and Zhenyu Chen. 2024. Source code summarization in the era of large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.07959 (2024)."},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2014.94"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME52107.2021.00018"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510205"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"crossref","unstructured":"Xin Xia Lingfeng Bao David Lo Zhenchang Xing Ahmed\u00a0E Hassan and Shanping Li. 2017. Measuring program comprehension: A large-scale field study with professionals. IEEE Transactions on Software Engineering 44 10 (2017) 951\u2013976.","DOI":"10.1109\/TSE.2017.2734091"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/552"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Xuejun Zhang Xia Hou Xiuming Qiao and Wenfeng Song. 2024. A review of automatic source code summarization. Empirical Software Engineering 29 6 (2024) 162.","DOI":"10.1007\/s10664-024-10553-6"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"crossref","unstructured":"Yuxia Zhang Zhiqing Qiu Klaas-Jan Stol Wenhui Zhu Jiaxin Zhu Yingchen Tian and Hui Liu. 2024. Automatic commit message generation: A critical review and directions for future work. IEEE Transactions on Software Engineering (2024).","DOI":"10.1109\/TSE.2024.3364675"},{"key":"e_1_3_3_2_32_2","unstructured":"Yuxiang Zhu and Minxue Pan. 2019. Automatic code summarization: A systematic literature review. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1909.04352 (2019)."}],"event":{"name":"LLM4Code '26: 3rd International Workshop on Large Language Models For Code","location":"Rio de Janeiro Brazil","acronym":"LLM4Code '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 3rd International Workshop on Large Language Models For Code"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786181.3788728","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:37:30Z","timestamp":1783406250000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786181.3788728"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":31,"alternative-id":["10.1145\/3786181.3788728","10.1145\/3786181"],"URL":"https:\/\/doi.org\/10.1145\/3786181.3788728","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-07-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}