{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,29]],"date-time":"2026-08-29T07:01:47Z","timestamp":1787986907599,"version":"build-2784847793"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770854.3785692","type":"proceedings-article","created":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T12:07:40Z","timestamp":1785499660000},"page":"2652-2662","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["REALM-Bench: A Benchmark for Evaluating Multi-Agent Systems on Real-world, Dynamic Planning and Scheduling Tasks"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-7869-4000","authenticated-orcid":false,"given":"Longling","family":"Geng","sequence":"first","affiliation":[{"name":"Computer Science Department, Stanford University, Stanford, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9860-2311","authenticated-orcid":false,"given":"Edward Y.","family":"Chang","sequence":"additional","affiliation":[{"name":"Computer Science Department, Stanford University, Stanford, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,4,20]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Proceedings of the International Conference on Autonomous Agents and Multiagent Systems (AAMAS). Virtual.","author":"Organizers ANAC","year":"2023","unstructured":"ANAC Organizers. 2023. Automated Negotiation Agents Competition 2023: Benchmarking Negotiation Strategies. In Proceedings of the International Conference on Autonomous Agents and Multiagent Systems (AAMAS). Virtual."},{"key":"e_1_3_2_2_2_1","unstructured":"Anthropic. 2024. The Claude 3 Model Family: Opus Sonnet Haiku. Technical Report. https:\/\/www.anthropic.com\/news\/claude-3-family Technical Report."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3749421"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.14778\/3750601.3750611"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/TII.2022.3167380"},{"key":"e_1_3_2_2_6_1","volume-title":"TimeBench: A Comprehensive Evaluation of Temporal Reasoning Abilities in Large Language Models. arXiv preprint arXiv:2311.17667","author":"Chu Zheng","year":"2023","unstructured":"Zheng Chu, Jingchang Chen, Qianglong Chen, Weijiang Yu, Haotian Wang, Ming Liu, and Bing Qin. 2023. TimeBench: A Comprehensive Evaluation of Temporal Reasoning Abilities in Large Language Models. arXiv preprint arXiv:2311.17667 (2023). https:\/\/arxiv.org\/abs\/2311.17667"},{"key":"e_1_3_2_2_7_1","unstructured":"DeepSeek-AI Daya Guo Dejian Yang and more. 2025. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. arXiv:2501.12948 [cs.CL] https:\/\/arxiv.org\/abs\/2501.12948"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0377--2217(97)00019--2"},{"key":"e_1_3_2_2_9_1","first-page":"389","article-title":"A Computational Study of Shifting Bottleneck Procedures for Shop Scheduling Problems","volume":"4","author":"Demirkol E.","year":"1998","unstructured":"E. Demirkol, S. Mehta, and R. Uzsoy. 1998. A Computational Study of Shifting Bottleneck Procedures for Shop Scheduling Problems. Journal of Heuristics 4, 4 (1998), 389--412.","journal-title":"Journal of Heuristics"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmsy.2023.01.004"},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the International Conference on Automated Planning and Scheduling (ICAPS). ICAPS, Virtual.","author":"Doe J.","unstructured":"J. Doe and R. Roe. 2022. The Dynamic Planning Competition 2022: Challenges and Results. In Proceedings of the International Conference on Automated Planning and Scheduling (ICAPS). ICAPS, Virtual."},{"key":"e_1_3_2_2_12_1","volume-title":"et al","author":"Team Gemini","year":"2023","unstructured":"Gemini Team, Rohan Anil, Sebastian Borgeaud, Yonghui Wu, et al . 2023. Gemini: A Family of Highly Capable Multimodal Models. arXiv preprint arXiv:2312.11805 (2023). https:\/\/arxiv.org\/abs\/2312.11805"},{"key":"e_1_3_2_2_13_1","volume-title":"Chang","author":"Geng Longling","year":"2025","unstructured":"Longling Geng and Edward Y. Chang. 2025. ALAS: Transactional and Dynamic Multi-Agent LLM Planning. arXiv preprint arXiv:2511.03094 (2025). https:\/\/arxiv.org\/abs\/2511.03094"},{"key":"e_1_3_2_2_14_1","volume-title":"Automatic Programming via Large Language Models with Population Self-Evolution for Dynamic Job Shop Scheduling Problem. arXiv:2410.22657","author":"Huang Jin","year":"2024","unstructured":"Jin Huang, Xinyu Li, Liang Gao, Qihao Liu, and Yue Teng. 2024. Automatic Programming via Large Language Models with Population Self-Evolution for Dynamic Job Shop Scheduling Problem. arXiv:2410.22657 (2024). arXiv:2410.22657 [cs.NE] https:\/\/arxiv.org\/abs\/2410.22657"},{"key":"e_1_3_2_2_15_1","volume-title":"et al","author":"Khattab Omar","year":"2023","unstructured":"Omar Khattab, Arnav Singhvi, Paridhi Maheshwari, Zhiyuan Zhang, et al . 2023. DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines. arXiv preprint arXiv:2310.03714 (2023). https:\/\/arxiv.org\/abs\/2310.03714"},{"key":"e_1_3_2_2_16_1","unstructured":"LangChain AI. 2024. LangGraph: Building Structured Applications with LLMs. https:\/\/github.com\/langchain-ai\/langgraph."},{"key":"e_1_3_2_2_17_1","volume-title":"Hani Itani, and more.","author":"Li Guohao","year":"2023","unstructured":"Guohao Li, Hasan Abed Al Kader Hammoud, Hani Itani, and more. 2023. CAMEL: Communicative Agents for ''Mind'' Exploration of Large Language Model Society. arXiv preprint arXiv:2303.17760 (2023)."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/TII.2024.3371489"},{"key":"e_1_3_2_2_19_1","unstructured":"Joao Moura. 2024. CrewAI Framework. https:\/\/github.com\/joaomdmoura\/crewAI."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cie.2013.05.023"},{"key":"e_1_3_2_2_21_1","unstructured":"OpenAI. 2024. Hello GPT-4o. https:\/\/openai.com\/index\/hello-gpt-4o\/. Accessed: 2025-01--30."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-2361-4"},{"key":"e_1_3_2_2_23_1","unstructured":"Yongliang Shen et al . 2023. TaskBench: Benchmarking large language models for task automation. arXiv preprint arXiv:2311.18760 (2023)."},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the International Conference on Automated Planning and Scheduling (ICAPS). ICAPS, Virtual.","author":"Smith A.","unstructured":"A. Smith and B. Johnson. 2020. The Process Planning Competition 2020: An Overview. In Proceedings of the International Conference on Automated Planning and Scheduling (ICAPS). ICAPS, Virtual."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.12169"},{"key":"e_1_3_2_2_26_1","volume-title":"PlanBench: An Extensible Benchmark for Evaluating Large Language Models on Planning and Reasoning about Change. arXiv preprint arXiv:2206.10498","author":"Valmeekam Karthik","year":"2023","unstructured":"Karthik Valmeekam, Matthew Marquez, Alberto Olmo, Sarath Sreedharan, and Subbarao Kambhampati. 2023. PlanBench: An Extensible Benchmark for Evaluating Large Language Models on Planning and Reasoning about Change. arXiv preprint arXiv:2206.10498 (2023). https:\/\/arxiv.org\/abs\/2206.10498"},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of the Conference on Language Modeling (COLM).","author":"Wu Qingyun","year":"2024","unstructured":"Qingyun Wu, Gagan Bansal, Jieyu Zhang, Yiran Wu, and Chi Wang. 2024. AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation. In Proceedings of the Conference on Language Modeling (COLM)."},{"key":"e_1_3_2_2_28_1","unstructured":"XAgent Team. 2023. XAgent: An Autonomous Agent for Complex Task Solving. https:\/\/github.com\/OpenBMB\/XAgent."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/3495724.3495861"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-03991-2_43"}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3770854.3785692","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,25]],"date-time":"2026-08-25T20:17:06Z","timestamp":1787689026000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770854.3785692"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,20]]},"references-count":30,"alternative-id":["10.1145\/3770854.3785692","10.1145\/3770854"],"URL":"https:\/\/doi.org\/10.1145\/3770854.3785692","relation":{},"subject":[],"published":{"date-parts":[[2026,4,20]]},"assertion":[{"value":"2026-04-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}