{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T15:39:03Z","timestamp":1788277143082,"version":"build-2803163510"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,8,8]],"date-time":"2026-08-08T00:00:00Z","timestamp":1786147200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770855.3817538","type":"proceedings-article","created":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T17:15:37Z","timestamp":1786036537000},"page":"9604-9615","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["SOP-Bench: Complex Industrial SOPs for Evaluating LLM Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-4855-2548","authenticated-orcid":false,"given":"Subhrangshu","family":"Nandi","sequence":"first","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3543-3829","authenticated-orcid":false,"given":"Arghya","family":"Datta","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-5009-8063","authenticated-orcid":false,"given":"Rohith","family":"Nama","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0254-8445","authenticated-orcid":false,"given":"Udita","family":"Patel","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-3388-9864","authenticated-orcid":false,"given":"Nikhil","family":"Vichare","sequence":"additional","affiliation":[{"name":"Amazon, Bellevue, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-5828-8627","authenticated-orcid":false,"given":"Indranil","family":"Bhattacharya","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-8958-1604","authenticated-orcid":false,"given":"Shivam","family":"Asija","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-5383-4561","authenticated-orcid":false,"given":"Arushi","family":"Gupta","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4310-0119","authenticated-orcid":false,"given":"Giuseppe","family":"Carenini","sequence":"additional","affiliation":[{"name":"University of British Columbia, Vancouver, BC, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7163-9161","authenticated-orcid":false,"given":"Jing","family":"Xu","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-3979-4251","authenticated-orcid":false,"given":"Shayan","family":"Ray","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1399-4835","authenticated-orcid":false,"given":"Huzefa","family":"Raja","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-8167-3238","authenticated-orcid":false,"given":"Aaron","family":"Chan","sequence":"additional","affiliation":[{"name":"Amazon, Bellevue, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-8096-5587","authenticated-orcid":false,"given":"Francesco","family":"Carbone","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2716-2339","authenticated-orcid":false,"given":"Esther Xu","family":"Fei","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2374-186X","authenticated-orcid":false,"given":"Gaoyuan","family":"Du","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0338-6549","authenticated-orcid":false,"given":"Zuhaib","family":"Akhtar","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4232-8499","authenticated-orcid":false,"given":"Prince","family":"Grover","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5287-1639","authenticated-orcid":false,"given":"Sreyoshi","family":"Bhaduri","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7325-1224","authenticated-orcid":false,"given":"Weian","family":"Chen","sequence":"additional","affiliation":[{"name":"Amazon, San Jose, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3944-5114","authenticated-orcid":false,"given":"Wei","family":"Zhang","sequence":"additional","affiliation":[{"name":"Amazon, San Jose, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1190-2698","authenticated-orcid":false,"given":"Ming","family":"Xiong","sequence":"additional","affiliation":[{"name":"Amazon, Santa Clara, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-3673-8058","authenticated-orcid":false,"given":"Harshita","family":"Asnani","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-7508-7861","authenticated-orcid":false,"given":"Jeetu","family":"Mirchandani","sequence":"additional","affiliation":[{"name":"Amazon, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,8,8]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Anthropic. 2024. Claude 3.5 Sonnet. https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet."},{"key":"e_1_3_2_1_2_1","unstructured":"Anthropic. 2025a. Claude 3.7 Sonnet and Claude Code. https:\/\/www.anthropic.com\/news\/claude-3-7-sonnet."},{"key":"e_1_3_2_1_3_1","unstructured":"Anthropic. 2025b. Introducing Claude 4. https:\/\/www.anthropic.com\/news\/claude-4."},{"key":"e_1_3_2_1_4_1","unstructured":"Anthropic. 2025c. Introducing Claude Opus 4.5. https:\/\/www.anthropic.com\/news\/claude-opus-4-5."},{"key":"e_1_3_2_1_5_1","unstructured":"Anthropic. 2025d. Introducing Claude Sonnet 4.5. https:\/\/www.anthropic.com\/news\/claude-sonnet-4-5."},{"key":"e_1_3_2_1_6_1","volume-title":"Benchagents: Automated benchmark creation with agent interaction.","author":"Butt Natasha","year":"2024","unstructured":"Natasha Butt, Varun Chandrasekaran, Neel Joshi, Besmira Nushi, and Vidhisha Balachandran. 2024. Benchagents: Automated benchmark creation with agent interaction."},{"key":"e_1_3_2_1_7_1","unstructured":"Harrison Chase. 2022. Langchain. https:\/\/github.com\/hwchase17\/langchain."},{"key":"e_1_3_2_1_8_1","volume-title":"Thien Huu Nguyen, and Yoshua Bengio","author":"Chevalier-Boisvert Maxime","year":"2018","unstructured":"Maxime Chevalier-Boisvert, Dzmitry Bahdanau, Salem Lahlou, Lucas Willems, Chitwan Saharia, Thien Huu Nguyen, and Yoshua Bengio. 2018. Babyai: First steps towards grounded language learning with a human in the loop."},{"key":"e_1_3_2_1_9_1","first-page":"563","article-title":"Standard operating procedures: A novel perspective","volume":"52","author":"Treville Suzanne De","year":"2009","unstructured":"Suzanne De Treville, John Antonakis, and Norman M. Edelson. 2009. Standard operating procedures: A novel perspective. Business Horizons, V52, 6 (2009), 563-572.","journal-title":"Business Horizons"},{"key":"e_1_3_2_1_10_1","unstructured":"DeepSeek. 2025. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. https:\/\/api-docs.deepseek.com\/news\/news250120."},{"key":"e_1_3_2_1_11_1","volume-title":"Jan Mendling, and Hajo A. Reijers.","author":"Dumas Marlon","year":"2018","unstructured":"Marlon Dumas, Marcello La Rosa, Jan Mendling, and Hajo A. Reijers. 2018. Fundamentals of business process management. Springer."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 2022 CHI Conference on Human Factors in Computing Systems. 1-19","author":"Gordon Mitchell L.","unstructured":"Mitchell L. Gordon, Michelle S. Lam, Ji Su Park, Kshitij Patel, Jeff Hancock, Tatsunori Hashimoto, and Michael S. Bernstein. 2022. Jury learning: Integrating dissenting voices into machine learning models. In Proceedings of the 2022 CHI Conference on Human Factors in Computing Systems. 1-19."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track. 1227-1238","author":"Gschwind Thomas","year":"2025","unstructured":"Thomas Gschwind, Shramona Chakraborty, Nitin Gupta, and Sameep Mehta. 2025. Classifier-Augmented Generation for Structured Workflow Prediction. In Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: Industry Track. 1227-1238."},{"key":"e_1_3_2_1_14_1","volume-title":"The Twelfth International Conference on Learning Representations (ICLR). URL: https:\/\/openreview.net\/forum?id=R0c2qtalgG.","author":"Huang Yue","year":"2024","unstructured":"Yue Huang, Jiawen Shi, Yuan Li, Chenrui Fan, Siyuan Wu, Qihui Zhang, Yixin Liu, Pan Zhou, Yao Wan, Neil Zhenqiang Gong, and Lichao Sun. 2024. Metatool benchmark for large language models: Deciding whether to use tools and which to use. In The Twelfth International Conference on Learning Representations (ICLR). URL: https:\/\/openreview.net\/forum?id=R0c2qtalgG."},{"key":"e_1_3_2_1_15_1","volume-title":"Followeval: A multi-dimensional benchmark for assessing the instruction-following capability of large language models. arXiv preprint arXiv:2311.09829","author":"Yimin Jing","year":"2023","unstructured":"Yimin Jing et al., 2023. Followeval: A multi-dimensional benchmark for assessing the instruction-following capability of large language models. arXiv preprint arXiv:2311.09829 (2023)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.187"},{"key":"e_1_3_2_1_17_1","volume-title":"Hashimoto","author":"Li Xuechen","year":"2023","unstructured":"Xuechen Li, Tianyi Zhang, Yann Dubois, Rohan Taori, Ishaan Gulrajani, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. 2023a. AlpacaEval: An Automatic Evaluator of Instruction-following Models."},{"key":"e_1_3_2_1_18_1","volume-title":"V45, 15","author":"Lindsay Ann","year":"2003","unstructured":"Ann Lindsay, Denise Downs, and Ken Lunn. 2003. Business processes\u2014attempts to find a definition. Information and software technology, V45, 15 (2003), 1015-1019."},{"key":"e_1_3_2_1_19_1","volume-title":"Agentbench: Evaluating llms as agents. arXiv preprint arXiv:2308.03688","author":"Liu Xiao","year":"2023","unstructured":"Xiao Liu, Hao Chen, Hanchen Chen, et al., 2023. Agentbench: Evaluating llms as agents. arXiv preprint arXiv:2308.03688 (2023)."},{"key":"e_1_3_2_1_20_1","unstructured":"Meta. 2024. Meta. Introducing llama 3.1: Our most capable models to date. https:\/\/ai.meta.com\/blog\/meta-llama-3-1\/."},{"key":"e_1_3_2_1_21_1","unstructured":"OpenAI. 2024. Chatgpt (april 2024 version). https:\/\/openai.com\/chatgpt."},{"key":"e_1_3_2_1_22_1","unstructured":"OpenAI. 2025. Introducing gpt-oss. https:\/\/openai.com\/index\/introducing-gpt-oss\/."},{"key":"e_1_3_2_1_23_1","volume-title":"Gorilla: Large language model connected with massive apis. In Advances in Neural Information Processing Systems.","author":"Patil Kartikeya","year":"2023","unstructured":"Kartikeya Patil, Yuan Tian, Xiangning Lin, Chen Lin, Kevin Lin, Xinying Liu, Eric Xing, Dawei Lu, Nazneen Rajani, Ranjit Krishnan, et al., 2023. Gorilla: Large language model connected with massive apis. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_2_1_24_1","volume-title":"Toolllm: Facilitating large language models to master 16000 real-world apis. arXiv preprint arXiv:2307.16789","author":"Qin Yujia","year":"2023","unstructured":"Yujia Qin, Shihao Liang, Yining Ye, Kunlun Zhu, Lan Yan, Yaxi Lu, Yankai Lin, Xin Cong, Xiangru Tang, Bill Qian, et al., 2023. Toolllm: Facilitating large language models to master 16000 real-world apis. arXiv preprint arXiv:2307.16789 (2023)."},{"key":"e_1_3_2_1_25_1","volume-title":"Infobench: Evaluating instruction following ability in large language models. arXiv preprint arXiv:2401.03601","author":"Qin Yiwei","year":"2024","unstructured":"Yiwei Qin, Xiang Ren, et al., 2024. Infobench: Evaluating instruction following ability in large language models. arXiv preprint arXiv:2401.03601 (2024)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"e_1_3_2_1_27_1","volume-title":"Behind the Screen: Content Moderation in the Shadows of Social Media","author":"Roberts Sarah T.","unstructured":"Sarah T. Roberts. 2019. Behind the Screen: Content Moderation in the Shadows of Social Media. Yale University Press."},{"key":"e_1_3_2_1_28_1","volume-title":"International Conference on Learning Representations.","author":"Shridhar Mohit","year":"2020","unstructured":"Mohit Shridhar, Xingdi Yuan, Marc-Alexandre Cote, Yonatan Bisk, Adam Trischler, and Matthew Hausknecht. 2020. Alfworld: Aligning text and embodied environments for interactive learning. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_29_1","volume-title":"Planbench: An extensible benchmark for evaluating large language models on planning and reasoning about change. Advances in Neural Information Processing Systems, V36","author":"Valmeekam Karthik","year":"2023","unstructured":"Karthik Valmeekam, Matthew Marquez, Alberto Olmo, Sarath Sreedharan, and Subbarao Kambhampati. 2023. Planbench: An extensible benchmark for evaluating large language models on planning and reasoning about change. Advances in Neural Information Processing Systems, V36 (2023), 38975-38987."},{"key":"e_1_3_2_1_30_1","volume-title":"Arthur H. M. Ter Hofstede, Bartek Kiepuszewski, and Alistair P. Barros.","author":"Wil M.","year":"2003","unstructured":"Wil M. P. Van Der Aalst, Arthur H. M. Ter Hofstede, Bartek Kiepuszewski, and Alistair P. Barros. 2003. Workflow patterns. Distributed and parallel databases, V14, 1 (2003), 5-51."},{"key":"e_1_3_2_1_31_1","volume-title":"Autogpt: An autonomous gpt-4 based agent system. arXiv preprint arXiv:2304.03277","author":"Feng Wang","year":"2023","unstructured":"Feng Wang et al., 2023. Autogpt: An autonomous gpt-4 based agent system. arXiv preprint arXiv:2304.03277 (2023)."},{"key":"e_1_3_2_1_32_1","volume-title":"SOP-Maze: Evaluating Large Language Models on Complicated Business Standard Operating Procedures. arXiv preprint arXiv:2510.08942","author":"Wang Jiaming","year":"2025","unstructured":"Jiaming Wang, Zhe Tang, Yilin Jin, Peng Ding, Xiaoyu Li, and Xuezhi Cao. 2025. SOP-Maze: Evaluating Large Language Models on Complicated Business Standard Operating Procedures. arXiv preprint arXiv:2510.08942 (2025)."},{"key":"e_1_3_2_1_33_1","volume-title":"Super-naturalinstructions: Generalization via declarative instructions on 1600 nlp tasks. arXiv preprint arXiv:2204.07705","author":"Yizhong Wang","year":"2022","unstructured":"Yizhong Wang et al., 2022. Super-naturalinstructions: Generalization via declarative instructions on 1600 nlp tasks. arXiv preprint arXiv:2204.07705 (2022)."},{"key":"e_1_3_2_1_34_1","unstructured":"Bosi Wen Shaohan Li Pengcheng He Weizhu Liu et al. 2024. Benchmarking complex instruction-following with multiple constraints composition. arXiv preprint arXiv:2407.03978 (2024)."},{"key":"e_1_3_2_1_35_1","volume-title":"Auto-gpt for online decision making: Benchmarks and additional opinions.(2023). URL https:\/\/arxiv.org\/abs\/2306.02224","author":"Yang Hui","year":"2023","unstructured":"Hui Yang, Sifu Yue, and Yunzhong He. 2023. Auto-gpt for online decision making: Benchmarks and additional opinions.(2023). URL https:\/\/arxiv.org\/abs\/2306.02224 (2023)."},{"key":"e_1_3_2_1_36_1","volume-title":"React: Synergizing reasoning and acting in language models. arXiv preprint arXiv:2210.03629","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik Narasimhan, and Yuan Cao. 2023. React: Synergizing reasoning and acting in language models. arXiv preprint arXiv:2210.03629 (2023)."},{"key":"e_1_3_2_1_37_1","volume-title":"Efficient action space navigation in large language models with a* search. arXiv preprint arXiv:2310.13227","author":"Zhuang Yuchen","year":"2023","unstructured":"Yuchen Zhuang, Xiang Chen, Tong Yu, Saayan Mitra, Victor Bursztyn, Ryan A. Rossi, Somdeb Sarkhel, and Chao Zhang. 2023. Toolchain*: Efficient action space navigation in large language models with a* search. arXiv preprint arXiv:2310.13227 (2023)."}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"deposited":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T01:23:29Z","timestamp":1786065809000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770855.3817538"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,8]]},"references-count":37,"alternative-id":["10.1145\/3770855.3817538","10.1145\/3770855"],"URL":"https:\/\/doi.org\/10.1145\/3770855.3817538","relation":{},"subject":[],"published":{"date-parts":[[2026,8,8]]},"assertion":[{"value":"2026-08-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}