{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,14]],"date-time":"2026-05-14T01:18:29Z","timestamp":1778721509969,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Research Council of Norway","award":["#288787"],"award-info":[{"award-number":["#288787"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3661167.3661171","type":"proceedings-article","created":{"date-parts":[[2024,6,14]],"date-time":"2024-06-14T12:24:25Z","timestamp":1718367865000},"page":"470-475","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Agent-Driven Automatic Software Improvement"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7213-3732","authenticated-orcid":false,"given":"Fernando","family":"Vallecillos Ruiz","sequence":"first","affiliation":[{"name":"Simula Research Laboratory, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,6,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"A. Alaboudi and T.\u00a0D. LaToza. 2021. An Exploratory Study of Debugging Episodes. arxiv:2105.02162\u00a0[cs]"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","unstructured":"R. Bavishi H. Joshi J.\u00a0P.\u00a0C. S\u00e1nchez A. Fariha S. Gulwani V. Le I. Radicek and A. Tiwari. 2022. Neurosymbolic Repair for Low-Code Formula Languages. https:\/\/doi.org\/10.48550\/arXiv.2207.11765 arxiv:2207.11765\u00a0[cs]","DOI":"10.48550\/arXiv.2207.11765"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"D. Chen H. Wang Y. Huo Y. Li and H. Zhang. 2023. GameGPT: Multi-agent Collaborative Framework for Game Development. https:\/\/doi.org\/10.48550\/arXiv.2310.08067 arxiv:2310.08067\u00a0[cs]","DOI":"10.48550\/arXiv.2310.08067"},{"key":"e_1_3_2_1_4_1","unstructured":"X. Chen M. Lin N. Sch\u00e4rli and D. Zhou. 2023. Teaching Large Language Models to Self-Debug. arxiv:2304.05128\u00a0[cs]"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","unstructured":"Z. Fan X. Gao M. Mirchev A. Roychoudhury and S.\u00a0H. Tan. 2023. Automated Repair of Programs from Large Language Models. https:\/\/doi.org\/10.48550\/arXiv.2205.10583 arxiv:2205.10583\u00a0[cs]","DOI":"10.48550\/arXiv.2205.10583"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"R. Feldt S. Kang J. Yoon and S. Yoo. 2023. Towards Autonomous Testing Agents via Conversational Large Language Models. arxiv:2306.05152\u00a0[cs]","DOI":"10.1109\/ASE56229.2023.00148"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"T. Guo X. Chen Y. Wang R. Chang S. Pei N.\u00a0V. Chawla O. Wiest and X. Zhang. 2024. Large Language Model Based Multi-Agents: A Survey of Progress and Challenges. arxiv:2402.01680\u00a0[cs]","DOI":"10.24963\/ijcai.2024\/890"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3623175"},{"key":"e_1_3_2_1_9_1","unstructured":"S. Hong X. Zheng J. Chen Y. Cheng J. Wang C. Zhang Z. Wang S.\u00a0K.\u00a0S. Yau Z. Lin L. Zhou C. Ran L. Xiao and C. Wu. 2023. MetaGPT: Meta Programming for Multi-Agent Collaborative Framework. arxiv:2308.00352\u00a0[cs]"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","unstructured":"M. Hort A. Grishina and L. Moonen. 2023. An Exploratory Literature Study on Sharing and Energy Use of Language Models for Source Code. https:\/\/doi.org\/10.48550\/arXiv.2307.02443 arxiv:2307.02443\u00a0[cs]","DOI":"10.48550\/arXiv.2307.02443"},{"key":"e_1_3_2_1_11_1","unstructured":"X. Huang W. Liu X. Chen X. Wang H. Wang D. Lian Y. Wang R. Tang and E. Chen. 2024. Understanding the Planning of LLM Agents: A Survey. arxiv:2402.02716\u00a0[cs]"},{"key":"e_1_3_2_1_12_1","volume-title":"Sleeper Agents: Training Deceptive LLMs That Persist Through Safety Training. arxiv:2401.05566\u00a0[cs]","author":"Hubinger E.","year":"2024","unstructured":"E. Hubinger, C. Denison, J. Mu, M. Lambert, M. Tong, M. MacDiarmid, T. Lanham, D.\u00a0M. Ziegler, T. Maxwell, N. Cheng, A. Jermyn, A. Askell, A. Radhakrishnan, C. Anil, D. Duvenaud, D. Ganguli, F. Barez, J. Clark, K. Ndousse, K. Sachan, M. Sellitto, M. Sharma, N. DasSarma, R. Grosse, S. Kravec, Y. Bai, Z. Witten, M. Favaro, J. Brauner, H. Karnofsky, P. Christiano, S.\u00a0R. Bowman, L. Graham, J. Kaplan, S. Mindermann, R. Greenblatt, B. Shlegeris, N. Schiefer, and E. Perez. 2024. Sleeper Agents: Training Deceptive LLMs That Persist Through Safety Training. arxiv:2401.05566\u00a0[cs]"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"T. Ji Y. Wu C. Wang X. Zhang and Z. Wang. 2018. The Coming Era of AlphaHacking? A Survey of Automatic Software Vulnerability Detection Exploitation and Patching Techniques. arxiv:1805.11001\u00a0[cs]","DOI":"10.1109\/DSC.2018.00017"},{"key":"e_1_3_2_1_14_1","volume-title":"F. Bressand, G. Lengyel, G. Lample, L. Saulnier, L.\u00a0R. Lavaud, M.-A. Lachaux, P. Stock, T.\u00a0L. Scao, T. Lavril, T. Wang, T. Lacroix, and W.\u00a0E. Sayed.","author":"Jiang Q.","year":"2023","unstructured":"A.\u00a0Q. Jiang, A. Sablayrolles, A. Mensch, C. Bamford, D.\u00a0S. Chaplot, D. de\u00a0las Casas, F. Bressand, G. Lengyel, G. Lample, L. Saulnier, L.\u00a0R. Lavaud, M.-A. Lachaux, P. Stock, T.\u00a0L. Scao, T. Lavril, T. Wang, T. Lacroix, and W.\u00a0E. Sayed. 2023. Mistral 7B. arxiv:2310.06825\u00a0[cs]"},{"key":"e_1_3_2_1_15_1","volume-title":"Research: Quantifying GitHub Copilot\u2019s Impact on Developer Productivity and Happiness.","author":"Kalliamvakou E.","year":"2022","unstructured":"E. Kalliamvakou. 2022. Research: Quantifying GitHub Copilot\u2019s Impact on Developer Productivity and Happiness."},{"key":"e_1_3_2_1_16_1","volume-title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. In Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 1).","author":"Lu S.","unstructured":"S. Lu, D. Guo, S. Ren, J. Huang, A. Svyatkovskiy, A. Blanco, C. Clement, D. Drain, D. Jiang, D. Tang, G. Li, L. Zhou, L. Shou, L. Zhou, M. Tufano, M. Gong, M. Zhou, N. Duan, N. Sundaresan, S.\u00a0K. Deng, S. Fu, and S. Liu. 2021. CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. In Thirty-Fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 1)."},{"key":"e_1_3_2_1_17_1","unstructured":"S. Ma H. Wang L. Ma L. Wang W. Wang S. Huang L. Dong R. Wang J. Xue and F. Wei. 2024. The Era of 1-Bit LLMs: All Large Language Models Are in 1.58 Bits. arxiv:2402.17764\u00a0[cs]"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.448"},{"key":"e_1_3_2_1_19_1","volume":"202","author":"Minaee S.","unstructured":"S. Minaee, T. Mikolov, N. Nikzad, M. Chenaghlu, R. Socher, X. Amatriain, and J. Gao. 2024. Large Language Models: A Survey. arxiv:2402.06196\u00a0[cs]","journal-title":"J. Gao."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/icse48619.2023.00180"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"H. Pearce B. Ahmad B. Tan B. Dolan-Gavitt and R. Karri. 2021. Asleep at the Keyboard? Assessing the Security of GitHub Copilot\u2019s Code Contributions. arxiv:2108.09293\u00a0[cs]","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_2_1_23_1","unstructured":"C. Qian X. Cong W. Liu C. Yang W. Chen Y. Su Y. Dang J. Li J. Xu D. Li Z. Liu and M. Sun. 2023. Communicative Agents for Software Development. arxiv:2307.07924\u00a0[cs]"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"A. Radhakrishnan K. Nguyen A. Chen C. Chen C. Denison D. Hernandez E. Durmus E. Hubinger J. Kernion K. Luko\u0161i\u016bt\u0117 N. Cheng N. Joseph N. Schiefer O. Rausch S. McCandlish S.\u00a0E. Showk T. Lanham T. Maxwell V. Chandrasekaran Z. Hatfield-Dodds J. Kaplan J. Brauner S.\u00a0R. Bowman and E. Perez. 2023. Question Decomposition Improves the Faithfulness of Model-Generated Reasoning. https:\/\/doi.org\/10.48550\/arXiv.2307.11768 arxiv:2307.11768\u00a0[cs]","DOI":"10.48550\/arXiv.2307.11768"},{"key":"e_1_3_2_1_25_1","unstructured":"T. Shen R. Jin Y. Huang C. Liu W. Dong Z. Guo X. Wu Y. Liu and D. Xiong. 2023. Large Language Model Alignment: A Survey. arxiv:2309.15025\u00a0[cs]"},{"key":"e_1_3_2_1_26_1","unstructured":"A. Shypula A. Madaan Y. Zeng U. Alon J. Gardner M. Hashemi G. Neubig P. Ranganathan O. Bastani and A. Yazdanbakhsh. 2023. Learning Performance-Improving Code Edits. arxiv:2302.07867\u00a0[cs]"},{"key":"e_1_3_2_1_27_1","unstructured":"J.\u00a0B. Simon D. Karkada N. Ghosh and M. Belkin. 2023. More Is Better in Modern Machine Learning: When Infinite Overparameterization Is Optimal and Overfitting Is Obligatory. arxiv:2311.14646\u00a0[cs stat]"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"Y. Talebirad and A. Nadiri. 2023. Multi-Agent Collaboration: Harnessing the Power of Intelligent LLM Agents. https:\/\/doi.org\/10.48550\/arXiv.2306.03314 arxiv:2306.03314\u00a0[cs]","DOI":"10.48550\/arXiv.2306.03314"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2310.16944"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ict4s55073.2022.00015"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","unstructured":"J. Wang Y. Huang C. Chen Z. Liu S. Wang and Q. Wang. 2023. Software Testing with Large Language Model: Survey Landscape and Vision. https:\/\/doi.org\/10.48550\/arXiv.2307.07221 arxiv:2307.07221\u00a0[cs]","DOI":"10.48550\/arXiv.2307.07221"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","unstructured":"L. Wang C. Ma X. Feng Z. Zhang H. Yang J. Zhang Z. Chen J. Tang X. Chen Y. Lin W.\u00a0X. Zhao Z. Wei and J.-R. Wen. 2023. A Survey on Large Language Model Based Autonomous Agents. https:\/\/doi.org\/10.48550\/arXiv.2308.11432 arxiv:2308.11432\u00a0[cs]","DOI":"10.48550\/arXiv.2308.11432"},{"key":"e_1_3_2_1_33_1","volume-title":"Software Engineering: An Introduction. Number\u00a06 in The Kluwer International Series in Software Engineering","author":"Wohlin C.","year":"2000","unstructured":"C. Wohlin, P. Runeson, M. Host, M.\u00a0C. Ohlsson, B. Regnell, and A. Wesslen. 2000. Experimentation in Software Engineering: An Introduction. Number\u00a06 in The Kluwer International Series in Software Engineering. Springer."},{"key":"e_1_3_2_1_34_1","unstructured":"Z. Xi W. Chen X. Guo W. He Y. Ding B. Hong M. Zhang J. Wang S. Jin E. Zhou R. Zheng X. Fan X. Wang L. Xiong Y. Zhou W. Wang C. Jiang Y. Zou X. Liu Z. Yin S. Dou R. Weng W. Cheng Q. Zhang W. Qin Y. Zheng X. Qiu X. Huang and T. Gui. 2023. The Rise and Potential of Large Language Model Based Agents: A Survey. arxiv:2309.07864\u00a0[cs]"},{"key":"e_1_3_2_1_35_1","unstructured":"C.\u00a0S. Xia and L. Zhang. 2023. Conversational Automated Program Repair. arxiv:2301.13246\u00a0[cs]"},{"key":"e_1_3_2_1_36_1","volume-title":"Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT. arxiv:2304.00385\u00a0[cs]","author":"Xia S.","year":"2023","unstructured":"C.\u00a0S. Xia and L. Zhang. 2023. Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT. arxiv:2304.00385\u00a0[cs]"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534390"},{"key":"e_1_3_2_1_38_1","unstructured":"Q. Zhang C. Fang Y. Ma W. Sun and Z. Chen. 2023. A Survey of Learning-based Automated Program Repair. arxiv:2301.03270\u00a0[cs]"},{"key":"e_1_3_2_1_39_1","unstructured":"S. Zhang L. Dong X. Li S. Zhang X. Sun S. Wang J. Li R. Hu T. Zhang F. Wu and G. Wang. 2023. Instruction Tuning for Large Language Models: A Survey. arxiv:2308.10792\u00a0[cs]"},{"key":"e_1_3_2_1_40_1","unstructured":"T. Zhang V. Kishore F. Wu K.\u00a0Q. Weinberger and Y. Artzi. 2020. BERTScore: Evaluating Text Generation with BERT. arxiv:1904.09675\u00a0[cs]"},{"key":"e_1_3_2_1_41_1","volume":"202","author":"Zheng Z.","unstructured":"Z. Zheng, K. Ning, Y. Wang, J. Zhang, D. Zheng, M. Ye, and J. Chen. 2024. A Survey of Large Language Models for Code: Evolution, Benchmarking, and Future Trends. arxiv:2311.10372\u00a0[cs]","journal-title":"J. Chen."},{"key":"e_1_3_2_1_42_1","volume":"202","author":"Zhou K.","unstructured":"K. Zhou, Y. Zhu, Z. Chen, W. Chen, W.\u00a0X. Zhao, X. Chen, Y. Lin, J.-R. Wen, and J. Han. 2023. Don\u2019t Make Your LLM an Evaluation Benchmark Cheater. arxiv:2311.01964\u00a0[cs]","journal-title":"J. Han."}],"event":{"name":"EASE 2024: 28th International Conference on Evaluation and Assessment in Software Engineering","location":"Salerno Italy","acronym":"EASE 2024"},"container-title":["Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3661167.3661171","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3661167.3661171","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T11:14:57Z","timestamp":1755861297000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3661167.3661171"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":42,"alternative-id":["10.1145\/3661167.3661171","10.1145\/3661167"],"URL":"https:\/\/doi.org\/10.1145\/3661167.3661171","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-06-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}