{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T07:17:21Z","timestamp":1783408641125,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":12,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2326576"],"award-info":[{"award-number":["2326576"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786181.3788708","type":"proceedings-article","created":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:28:24Z","timestamp":1783405704000},"page":"1-5","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Continuous Benchmark Generation for Evaluating Enterprise-scale LLM Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7568-0624","authenticated-orcid":false,"given":"Divyanshu","family":"Saxena","sequence":"first","affiliation":[{"name":"UT Austin, Austin, Texas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-4496-0203","authenticated-orcid":false,"given":"Rishikesh","family":"Maurya","sequence":"additional","affiliation":[{"name":"Microsoft, Bengaluru, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6667-4321","authenticated-orcid":false,"given":"Xiaoxuan","family":"Ou","sequence":"additional","affiliation":[{"name":"Microsoft, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6949-8685","authenticated-orcid":false,"given":"Gagan","family":"Somashekar","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, Washington, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1332-690X","authenticated-orcid":false,"given":"Shachee Mishra","family":"Gupta","sequence":"additional","affiliation":[{"name":"Microsoft, Noida, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7377-7599","authenticated-orcid":false,"given":"Arun","family":"Iyer","sequence":"additional","affiliation":[{"name":"Microsoft, Bengaluru, India"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-1735-5876","authenticated-orcid":false,"given":"Yu","family":"Kang","sequence":"additional","affiliation":[{"name":"Microsoft, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0102-8139","authenticated-orcid":false,"given":"Chetan","family":"Bansal","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, Washington, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5920-170X","authenticated-orcid":false,"given":"Aditya","family":"Akella","sequence":"additional","affiliation":[{"name":"UT Austin, Austin, Texas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2019-213X","authenticated-orcid":false,"given":"Saravan","family":"Rajmohan","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, Washington, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,6]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"2024. SWE-bench Lite: A Lightweight Benchmark for Evaluating LLMs on Software Engineering Tasks. https:\/\/www.swebench.com\/lite.html."},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3766882.3767182"},{"key":"e_1_3_3_1_4_2","unstructured":"Naman Gupta Shashank Kirtania Priyanshu Gupta Krishna Kariya Sumit Gulwani Arun Iyer Suresh Parthasarathy Arjun Radhakrishna Sriram\u00a0K. Rajamani and Gustavo Soares. 2024. STACKFEED: Structured Textual Actor-Critic Knowledge Base Editing with FeedBack. arxiv:https:\/\/arXiv.org\/abs\/2410.10584\u00a0[cs.AI]"},{"key":"e_1_3_3_1_5_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Jimenez Carlos\u00a0E","year":"2024","unstructured":"Carlos\u00a0E Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik\u00a0R Narasimhan. 2024. SWE-bench: Can Language Models Resolve Real-world Github Issues?. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_3_1_6_2","volume-title":"OSDI\u201925: 19th USENIX Symposium on Operating Systems Design and Implementation","author":"Lou Chang","year":"2025","unstructured":"Chang Lou, Dimas\u00a0Shidqi Parikesit, Yujin Huang, Zhewen Yang, Senapati Diwangkara, Yuzhuo Jing, Achmad\u00a0Imam Kistijantoro, Ding Yuan, Suman Nath, and Peng Huang. 2025. Deriving Semantic Checkers from Tests to Detect Silent Failures in Production Distributed Systems. In OSDI\u201925: 19th USENIX Symposium on Operating Systems Design and Implementation."},{"key":"e_1_3_3_1_7_2","unstructured":"OpenAI. 2024. Introducing SWE-bench Verified. https:\/\/openai.com\/index\/introducing-swe-bench-verified\/."},{"key":"e_1_3_3_1_8_2","unstructured":"Xi Wang Taketomo Isazawa Liana Mikaelyan and James Hensman. 2025. KBLaM: Knowledge Base augmented Language Model. arxiv:https:\/\/arXiv.org\/abs\/2410.10450\u00a0[cs.AI]"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE62328.2024.00056"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.52202\/079017-4037"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"crossref","unstructured":"Chunqiu\u00a0Steven Xia Yinlin Deng Soren Dunn and Lingming Zhang. 2025. Demystifying LLM-Based Software Engineering Agents. Proc. ACM Softw. Eng. 2 FSE (June 2025).","DOI":"10.1145\/3715754"},{"key":"e_1_3_3_1_12_2","unstructured":"Weiyuan Xu Juntao Luo Tao Huang Kaixin Sui Jie Geng Qijun Ma Isami Akasaka Xiaoxue Shi Jing Tang and Peng Cai. 2025. LogSage: An LLM-Based Framework for CI\/CD Failure Detection and Remediation with Industrial Validation. arxiv:https:\/\/arXiv.org\/abs\/2506.03691\u00a0[cs.SE]"},{"key":"e_1_3_3_1_13_2","unstructured":"Zhaoyang Yu Aoyang Fang Minghua Ma Chaoyun Zhang Ze Li Murali Chintalapati Xuchao Zhang Rujia Wang Chetan Bansal Saravan Rajmohan Qingwei Lin and et.al. 2025. Triangle: Empowering Incident Triage with Multi-Agent."}],"event":{"name":"LLM4Code '26: 3rd International Workshop on Large Language Models For Code","location":"Rio de Janeiro Brazil","acronym":"LLM4Code '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 3rd International Workshop on Large Language Models For Code"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786181.3788708","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786181.3788708","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T06:38:45Z","timestamp":1783406325000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786181.3788708"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":12,"alternative-id":["10.1145\/3786181.3788708","10.1145\/3786181"],"URL":"https:\/\/doi.org\/10.1145\/3786181.3788708","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-07-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}