{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,2]],"date-time":"2026-10-02T12:49:33Z","timestamp":1790945373342,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,5,26]],"date-time":"2026-05-26T00:00:00Z","timestamp":1779753600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,5,26]]},"DOI":"10.1145\/3786335.3813153","type":"proceedings-article","created":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:16:22Z","timestamp":1779419782000},"page":"584-596","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Willful Disobedience: Automatically Detecting Failures in Agentic Traces"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-2678-9190","authenticated-orcid":false,"given":"Reshabh K","family":"Sharma","sequence":"first","affiliation":[{"name":"University of Washington, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4573-9296","authenticated-orcid":false,"given":"Shraddha","family":"Barke","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9719-9842","authenticated-orcid":false,"given":"Benjamin","family":"Zorn","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,5,26]]},"reference":[{"key":"e_1_3_3_1_2_2","doi-asserted-by":"publisher","unstructured":"Mohamad Abou\u00a0Ali Fadi Dornaika and Jinan Charafeddine. 2025. Agentic AI: a comprehensive survey of architectures applications and future directions. Artificial Intelligence Review 59 1 (Nov. 2025). 10.1007\/s10462-025-11422-4","DOI":"10.1007\/s10462-025-11422-4"},{"key":"e_1_3_3_1_3_2","unstructured":"Yuntao Bai Saurav Kadavath Sandipan Kundu Amanda Askell Jackson Kernion Andy Jones Anna Chen Anna Goldie Azalia Mirhoseini Cameron McKinnon Carol Chen Catherine Olsson Christopher Olah Danny Hernandez Dawn Drain Deep Ganguli Dustin Li Eli Tran-Johnson Ethan Perez Jamie Kerr Jared Mueller Jeffrey Ladish Joshua Landau Kamal Ndousse Kamile Lukosuite Liane Lovitt Michael Sellitto Nelson Elhage Nicholas Schiefer Noemi Mercado Nova DasSarma Robert Lasenby Robin Larson Sam Ringer Scott Johnston Shauna Kravec Sheer\u00a0El Showk Stanislav Fort Tamera Lanham Timothy Telleen-Lawton Tom Conerly Tom Henighan Tristan Hume Samuel\u00a0R. Bowman Zac Hatfield-Dodds Ben Mann Dario Amodei Nicholas Joseph Sam McCandlish Tom Brown and Jared Kaplan. 2022. Constitutional AI: Harmlessness from AI Feedback. (2022). arxiv:https:\/\/arXiv.org\/abs\/2212.08073\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2212.08073"},{"key":"e_1_3_3_1_4_2","unstructured":"Shraddha Barke Arnav Goyal Alind Khare Avaljot Singh Suman Nath and Chetan Bansal. 2026. AgentRx: Diagnosing AI Agent Failures from Execution Trajectories. arxiv:https:\/\/arXiv.org\/abs\/2602.02475\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2602.02475"},{"key":"e_1_3_3_1_5_2","unstructured":"Victor Barres Honghua Dong Soham Ray Xujie Si and Karthik Narasimhan. 2025. \u03c42-Bench: Evaluating Conversational Agents in a Dual-Control Environment. arxiv:https:\/\/arXiv.org\/abs\/2506.07982\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2506.07982"},{"key":"e_1_3_3_1_6_2","unstructured":"Xinyun Chen Maxwell Lin Nathanael Sch\u00e4rli and Denny Zhou. 2023. Teaching Large Language Models to Self-Debug. (2023). arxiv:https:\/\/arXiv.org\/abs\/2304.05128\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2304.05128"},{"key":"e_1_3_3_1_7_2","unstructured":"Darshan Deshpande Varun Gangal Hersh Mehta Jitin Krishnan Anand Kannappan and Rebecca Qian. 2025. TRAIL: Trace Reasoning and Agentic Issue Localization. (2025). arxiv:https:\/\/arXiv.org\/abs\/2505.08638\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2505.08638"},{"key":"e_1_3_3_1_8_2","unstructured":"GitHub. 2026. GitHub Copilot: Your AI pair programmer. https:\/\/github.com\/features\/copilot. Product page for GitHub Copilot."},{"key":"e_1_3_3_1_9_2","unstructured":"Tommy Guy Peli de Halleux Reshabh\u00a0K Sharma and Ben Zorn. 2024. Prompts are Programs. https:\/\/blog.sigplan.org\/2024\/10\/22\/prompts-are-programs\/. SIGPLAN Perspectives Blog [Accessed 28-02-2026]."},{"key":"e_1_3_3_1_10_2","unstructured":"Sirui Hong Mingchen Zhuge Jiaqi Chen Xiawu Zheng Yuheng Cheng Ceyao Zhang Jinlin Wang Zili Wang Steven Ka\u00a0Shing Yau Zijuan Lin Liyang Zhou Chenyu Ran Lingfeng Xiao Chenglin Wu and J\u00fcrgen Schmidhuber. 2024. MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework. arxiv:https:\/\/arXiv.org\/abs\/2308.00352\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2308.00352"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin and Ting Liu. 2025. A Survey on Hallucination in Large Language Models: Principles Taxonomy Challenges and Open Questions. ACM Transactions on Information Systems 43 2 (Jan. 2025) 1\u201355. 10.1145\/37031553703155\"\/>","DOI":"10.1145\/3703155"},{"key":"e_1_3_3_1_12_2","unstructured":"Carlos\u00a0E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press and Karthik Narasimhan. 2024. SWE-bench: Can Language Models Resolve Real-World GitHub Issues? arxiv:https:\/\/arXiv.org\/abs\/2310.06770\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2310.06770"},{"key":"e_1_3_3_1_13_2","unstructured":"Omar Khattab Arnav Singhvi Paridhi Maheshwari Zhiyuan Zhang Keshav Santhanam Sri Vardhamanan Saiful Haq Ashutosh Sharma Thomas\u00a0T. Joshi Hanna Moazam Heather Miller Matei Zaharia and Christopher Potts. 2023. DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines. arxiv:https:\/\/arXiv.org\/abs\/2310.03714\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2310.03714"},{"key":"e_1_3_3_1_14_2","unstructured":"LangChain. 2024. LangSmith: AI Agent & LLM Observability Platform. https:\/\/www.langchain.com\/langsmith\/observability. Product documentation. Accessed 24-03-2026."},{"key":"e_1_3_3_1_15_2","unstructured":"Xiao Liu Hao Yu Hanchen Zhang Yifan Xu Xuanyu Lei Hanyu Lai Yu Gu Hangliang Ding Kaiwen Men Kejuan Yang Shudan Zhang Xiang Deng Aohan Zeng Zhengxiao Du Chenhui Zhang Sheng Shen Tianjun Zhang Yu Su Huan Sun Minlie Huang Yuxiao Dong and Jie Tang. 2024. AgentBench: Evaluating LLMs as Agents. arxiv:https:\/\/arXiv.org\/abs\/2308.03688\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2308.03688"},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"publisher","unstructured":"B. Meyer. 1992. Applying \u201cdesign by contract\u201d. Computer 25 10 (1992) 40\u201351. 10.1109\/2.161279","DOI":"10.1109\/2.161279"},{"key":"e_1_3_3_1_18_2","unstructured":"Gr\u00e9goire Mialon Cl\u00e9mentine Fourrier Craig Swift Thomas Wolf Yann LeCun and Thomas Scialom. 2023. GAIA: a benchmark for General AI Assistants. arxiv:https:\/\/arXiv.org\/abs\/2311.12983\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2311.12983"},{"key":"e_1_3_3_1_19_2","unstructured":"Microsoft. 2026. Microsoft 365 Copilot: Your everyday AI companion. https:\/\/www.microsoft.com\/en-us\/microsoft-365\/copilot. Product page for Microsoft 365 Copilot."},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Sewon Min Kalpesh Krishna Xinxi Lyu Mike Lewis Wen tau Yih Pang\u00a0Wei Koh Mohit Iyyer Luke Zettlemoyer and Hannaneh Hajishirzi. 2023. FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation. arxiv:https:\/\/arXiv.org\/abs\/2305.14251\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2305.14251","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"e_1_3_3_1_21_2","unstructured":"Joon\u00a0Sung Park Joseph\u00a0C. O\u2019Brien Carrie\u00a0J. Cai Meredith\u00a0Ringel Morris Percy Liang and Michael\u00a0S. Bernstein. 2023. Generative Agents: Interactive Simulacra of Human Behavior. arxiv:https:\/\/arXiv.org\/abs\/2304.03442\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2304.03442"},{"key":"e_1_3_3_1_22_2","series-title":"Proceedings of Machine Learning Research","first-page":"48371","volume-title":"Proceedings of the 42nd International Conference on Machine Learning","volume":"267","author":"Patil Shishir\u00a0G","year":"2025","unstructured":"Shishir\u00a0G Patil, Huanzhi Mao, Fanjia Yan, Charlie Cheng-Jie Ji, Vishnu Suresh, Ion Stoica, and Joseph\u00a0E. Gonzalez. 2025. The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation of Large Language Models. In Proceedings of the 42nd International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0267), Aarti Singh, Maryam Fazel, Daniel Hsu, Simon Lacoste-Julien, Felix Berkenkamp, Tegan Maharaj, Kiri Wagstaff, and Jerry Zhu (Eds.). PMLR, 48371\u201348392. https:\/\/proceedings.mlr.press\/v267\/patil25a.html"},{"key":"e_1_3_3_1_23_2","unstructured":"Shishir\u00a0G. Patil Tianjun Zhang Xin Wang and Joseph\u00a0E. Gonzalez. 2023. Gorilla: Large Language Model Connected with Massive APIs. arxiv:https:\/\/arXiv.org\/abs\/2305.15334\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2305.15334"},{"key":"e_1_3_3_1_24_2","unstructured":"Yujia Qin Shihao Liang Yining Ye Kunlun Zhu Lan Yan Yaxi Lu Yankai Lin Xin Cong Xiangru Tang Bill Qian Sihan Zhao Lauren Hong Runchu Tian Ruobing Xie Jie Zhou Mark Gerstein Dahai Li Zhiyuan Liu and Maosong Sun. 2023. ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs. (2023). arxiv:https:\/\/arXiv.org\/abs\/2307.16789\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2307.16789"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.230"},{"key":"e_1_3_3_1_26_2","unstructured":"Marco\u00a0Tulio Ribeiro Tongshuang Wu Carlos Guestrin and Sameer Singh. 2020. Beyond Accuracy: Behavioral Testing of NLP models with CheckList. arxiv:https:\/\/arXiv.org\/abs\/2005.04118\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2005.04118"},{"key":"e_1_3_3_1_27_2","unstructured":"Timo Schick Jane Dwivedi-Yu Roberto Dess\u00ec Roberta Raileanu Maria Lomeli Luke Zettlemoyer Nicola Cancedda and Thomas Scialom. 2023. Toolformer: Language Models Can Teach Themselves to Use Tools. arxiv:https:\/\/arXiv.org\/abs\/2302.04761\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2302.04761"},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1109\/spw63631.2024.00017"},{"key":"e_1_3_3_1_29_2","unstructured":"Reshabh\u00a0K Sharma Vinayak Gupta and Dan Grossman. 2024. SPML: A DSL for Defending Language Models Against Prompt Attacks. arxiv:https:\/\/arXiv.org\/abs\/2402.11755\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2402.11755"},{"key":"e_1_3_3_1_30_2","unstructured":"Reshabh\u00a0K Sharma Jonathan\u00a0De Halleux Shraddha Barke Dan Grossman and Benjamin Zorn. 2026. PromptPex: Automatic Test Generation for Language Model Prompts. arxiv:https:\/\/arXiv.org\/abs\/2503.05070\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2503.05070v1"},{"key":"e_1_3_3_1_31_2","unstructured":"Noah Shinn Federico Cassano Edward Berman Ashwin Gopinath Karthik Narasimhan and Shunyu Yao. 2023. Reflexion: Language Agents with Verbal Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2303.11366\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2303.11366"},{"key":"e_1_3_3_1_32_2","unstructured":"Jared Spataro. 2025. Microsoft Ignite 2025: Copilot and agents built to power the Frontier Firm. https:\/\/www.microsoft.com\/en-us\/microsoft-365\/blog\/2025\/11\/18\/microsoft-ignite-2025-copilot-and-agents-built-to-power-the-frontier-firm\/. Microsoft 365 Blog November 18 2025. Includes industry projections (e.g. IDC on enterprise agent adoption) cited in this paper."},{"key":"e_1_3_3_1_33_2","unstructured":"Ion Stoica Matei Zaharia Joseph Gonzalez Ken Goldberg Koushik Sen Hao Zhang Anastasios Angelopoulos Shishir\u00a0G. Patil Lingjiao Chen Wei-Lin Chiang and Jared\u00a0Q. Davis. 2024. Specifications: The missing link to making the development of LLM systems an engineering discipline. arxiv:https:\/\/arXiv.org\/abs\/2412.05299\u00a0[cs.SE] https:\/\/arxiv.org\/abs\/2412.05299"},{"key":"e_1_3_3_1_34_2","unstructured":"Peiyi Wang Lei Li Liang Chen Zefan Cai Dawei Zhu Binghuai Lin Yunbo Cao Qi Liu Tianyu Liu and Zhifang Sui. 2023. Large Language Models are not Fair Evaluators. (2023). arxiv:https:\/\/arXiv.org\/abs\/2305.17926\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2305.17926"},{"key":"e_1_3_3_1_35_2","unstructured":"Jason Wei Xuezhi Wang Dale Schuurmans Maarten Bosma Brian Ichter Fei Xia Ed Chi Quoc Le and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2201.11903\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2201.11903"},{"key":"e_1_3_3_1_36_2","unstructured":"Qingyun Wu Gagan Bansal Jieyu Zhang Yiran Wu Beibin Li Erkang Zhu Li Jiang Xiaoyun Zhang Shaokun Zhang Jiale Liu Ahmed\u00a0Hassan Awadallah Ryen\u00a0W White Doug Burger and Chi Wang. 2023. AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation. arxiv:https:\/\/arXiv.org\/abs\/2308.08155\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2308.08155"},{"key":"e_1_3_3_1_37_2","unstructured":"Tianbao Xie Danyang Zhang Jixuan Chen Xiaochuan Li Siheng Zhao Ruisheng Cao Toh\u00a0Jing Hua Zhoujun Cheng Dongchan Shin Fangyu Lei Yitao Liu Yiheng Xu Shuyan Zhou Silvio Savarese Caiming Xiong Victor Zhong and Tao Yu. 2024. OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments(NeurIPS 2024). arxiv:https:\/\/arXiv.org\/abs\/2404.07972\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2404.07972"},{"key":"e_1_3_3_1_38_2","unstructured":"Shunyu Yao Jeffrey Zhao Dian Yu Nan Du Izhak Shafran Karthik Narasimhan and Yuan Cao. 2023. ReAct: Synergizing Reasoning and Acting in Language Models. arxiv:https:\/\/arXiv.org\/abs\/2210.03629\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2210.03629"},{"key":"e_1_3_3_1_39_2","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric\u00a0P. Xing Hao Zhang Joseph\u00a0E. Gonzalez and Ion Stoica. 2023. Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena. arxiv:https:\/\/arXiv.org\/abs\/2306.05685\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2306.05685"},{"key":"e_1_3_3_1_40_2","unstructured":"Shuyan Zhou Frank\u00a0F. Xu Hao Zhu Xuhui Zhou Robert Lo Abishek Sridhar Xianyi Cheng Tianyue Ou Yonatan Bisk Daniel Fried Uri Alon and Graham Neubig. 2024. WebArena: A Realistic Web Environment for Building Autonomous Agents. arxiv:https:\/\/arXiv.org\/abs\/2307.13854\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2307.13854"}],"event":{"name":"CAIS '26: ACM Conference on AI and Agentic Systems","location":"San Jose CA USA","acronym":"CAIS '26"},"container-title":["Proceedings of the ACM Conference on AI and Agentic Systems"],"original-title":[],"deposited":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:35:55Z","timestamp":1779420955000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786335.3813153"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,26]]},"references-count":39,"alternative-id":["10.1145\/3786335.3813153","10.1145\/3786335"],"URL":"https:\/\/doi.org\/10.1145\/3786335.3813153","relation":{},"subject":[],"published":{"date-parts":[[2026,5,26]]},"assertion":[{"value":"2026-05-26","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}