{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T12:19:29Z","timestamp":1779365969606,"version":"3.53.0"},"publisher-location":"New York, NY, USA","reference-count":21,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"CNPq-Brazil","award":["404406\/2023-8"],"award-info":[{"award-number":["404406\/2023-8"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786167.3788430","type":"proceedings-article","created":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T11:40:19Z","timestamp":1779363619000},"page":"120-124","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["A Catalogue of Evaluation Metrics for LLM-Based Multi-Agent Frameworks in Software Engineering"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-2443-5188","authenticated-orcid":false,"given":"Ingrid","family":"Lima","sequence":"first","affiliation":[{"name":"State University of Cear\u00e1, Brazil, Fortaleza, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-3377-4573","authenticated-orcid":false,"given":"Vitor","family":"Linhares","sequence":"additional","affiliation":[{"name":"State University of Cear\u00e1, Brazil, Fortaleza, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-3910-3707","authenticated-orcid":false,"given":"Anderson Martins","family":"Gomes","sequence":"additional","affiliation":[{"name":"State University of Cear\u00e1, Brazil, Fortaleza, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6683-6869","authenticated-orcid":false,"given":"Paulo Henrique","family":"Maia","sequence":"additional","affiliation":[{"name":"State University of Ceara, Brazil, Fortaleza, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,5,21]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-acl.541"},{"key":"e_1_3_3_2_3_2","unstructured":"Alireza Ghafarollahi and Markus\u00a0J. Buehler. 2024. AtomAgents: Alloy design and discovery through physics-aware multi-modal multi-agent artificial intelligence. arxiv:https:\/\/arXiv.org\/abs\/2407.10022\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2407.10022"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Kanika Goswami Puneet Mathur Ryan Rossi and Franck Dernoncourt. 2025. Plotgen: Multi-agent llm-based scientific data visualization via multimodal feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2502.00988 (2025).","DOI":"10.1145\/3701716.3716888"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"crossref","unstructured":"Junda He Christoph Treude and David Lo. 2025. LLM-Based Multi-Agent Systems for Software Engineering: Literature Review Vision and the Road Ahead. ACM Transactions on Software Engineering and Methodology 34 5 (2025) 1\u201330.","DOI":"10.1145\/3712003"},{"key":"e_1_3_3_2_6_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Hong Sirui","year":"2023","unstructured":"Sirui Hong, Mingchen Zhuge, Jonathan Chen, Xiawu Zheng, Yuheng Cheng, Jinlin Wang, Ceyao Zhang, Zili Wang, Steven Ka\u00a0Shing Yau, Zijuan Lin, et\u00a0al. 2023. MetaGPT: Meta programming for a multi-agent collaborative framework. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_3_2_7_2","unstructured":"Yoichi Ishibashi and Yoshimasa Nishimura. 2024. Self-organized agents: A llm multi-agent framework toward ultra large-scale code generation and optimization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2404.02183 (2024)."},{"key":"e_1_3_3_2_8_2","unstructured":"Sourena Khanzadeh. 2025. AgentMesh: A Cooperative Multi-Agent Generative AI Framework for Software Development Automation. (2025). arxiv:https:\/\/arXiv.org\/abs\/2507.19902\u00a0[cs.SE]"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.1105"},{"key":"e_1_3_3_2_10_2","unstructured":"Leilei Lin Yingming Zhou Wenlong Chen and Chen Qian. 2024. Think-on-process: Dynamic process generation for collaborative development of multi-agent system. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.06568 (2024)."},{"key":"e_1_3_3_2_11_2","unstructured":"Yuxuan Liu Hongda Sun Wei Liu Jian Luan Bo Du and Rui Yan. 2025. MobileSteward: Integrating Multiple App-Oriented Agents with Self-Evolution to Automate Cross-App Instructions. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2502.16796 (2025)."},{"key":"e_1_3_3_2_12_2","unstructured":"Yohei Nakajima. 2023. BabyAGI. https:\/\/github.com\/yoheinakajima\/babyagi. GitHub repository."},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/Forge66646.2025.00026"},{"key":"e_1_3_3_2_14_2","unstructured":"Timothy Ossowski Jixuan Chen Danyal Maqbool Zefan Cai Tyler Bradshaw and Junjie Hu. 2024. Comma: A communicative multimodal multi-agent benchmark. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.07553 (2024)."},{"key":"e_1_3_3_2_15_2","unstructured":"Joon\u00a0Sung Park Joseph\u00a0C. O\u2019Brien Carrie\u00a0J. Cai Meredith\u00a0Ringel Morris Percy Liang and Michael\u00a0S. Bernstein. 2023. Generative Agents: Interactive Simulacra of Human Behavior. arxiv:https:\/\/arXiv.org\/abs\/2304.03442\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2304.03442"},{"key":"e_1_3_3_2_16_2","unstructured":"Chen Qian Jiahao Li Yufan Dang Wei Liu YiFei Wang Zihao Xie Weize Chen Cheng Yang Yingli Zhang Zhiyuan Liu et\u00a0al. 2024. Iterative experience refinement of software-developing agents. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.04219 (2024)."},{"key":"e_1_3_3_2_17_2","unstructured":"Chen Qian Wei Liu Hongzhang Liu Nuo Chen Yufan Dang Jiahao Li Cheng Yang Weize Chen Yusheng Su Xin Cong et\u00a0al. 2023. Chatdev: Communicative agents for software development. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.07924 (2023)."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-acl.259"},{"key":"e_1_3_3_2_19_2","volume-title":"First Conference on Language Modeling","author":"Wu Qingyun","year":"2024","unstructured":"Qingyun Wu, Gagan Bansal, Jieyu Zhang, Yiran Wu, Beibin Li, Erkang Zhu, Li Jiang, Xiaoyun Zhang, Shaokun Zhang, Jiale Liu, et\u00a0al. 2024. Autogen: Enabling next-gen LLM applications via multi-agent conversations. In First Conference on Language Modeling."},{"key":"e_1_3_3_2_20_2","unstructured":"Hui Yang Sifu Yue and Yunzhong He. 2023. Auto-GPT for Online Decision Making: Benchmarks and Additional Opinions. arxiv:https:\/\/arXiv.org\/abs\/2306.02224\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2306.02224"},{"key":"e_1_3_3_2_21_2","unstructured":"Rui Ye Shuo Tang Rui Ge Yaxin Du Zhenfei Yin Siheng Chen and Jing Shao. 2025. MAS-GPT: Training LLMs to build LLM-based multi-agent systems. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2503.03686 (2025)."},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"crossref","unstructured":"Sai Zhang Zhenchang Xing Ronghui Guo Fangzhou Xu Lei Chen Zhaoyuan Zhang Xiaowang Zhang Zhiyong Feng and Zhiqiang Zhuang. 2025. Empowering agile-based generative software development through human-ai teamwork. ACM Transactions on Software Engineering and Methodology (2025).","DOI":"10.1145\/3702987"}],"event":{"name":"AGENT '26: International Workshop on Agentic Engineering","location":"Rio de Janeiro Brazil","acronym":"AGENT '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 2026 International Workshop on Agentic Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786167.3788430","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T12:02:27Z","timestamp":1779364947000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786167.3788430"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":21,"alternative-id":["10.1145\/3786167.3788430","10.1145\/3786167"],"URL":"https:\/\/doi.org\/10.1145\/3786167.3788430","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-05-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}