{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T10:05:02Z","timestamp":1781690702449,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":33,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,8,3]]},"DOI":"10.1145\/3711896.3737390","type":"proceedings-article","created":{"date-parts":[[2025,8,3]],"date-time":"2025-08-03T21:03:27Z","timestamp":1754255007000},"page":"5810-5818","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["A Framework for Evaluating AI Agents in Open-Ended Conversations via Scripted Simulation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-7589-4670","authenticated-orcid":false,"given":"Clarice","family":"Wang","sequence":"first","affiliation":[{"name":"University of Pennsylvania, Philadelphia, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3375-4602","authenticated-orcid":false,"given":"Yimin","family":"Shi","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0914-4580","authenticated-orcid":false,"given":"Xiaokui","family":"Xiao","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,8,3]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Marah Abdin et al. Phi-4 Technical Report. 2024. arXiv: 2412.08905 [cs.CL]. url: https:\/\/arxiv.org\/abs\/2412.08905."},{"key":"e_1_3_2_2_2_1","volume-title":"Training a helpful and harmless assistant with reinforcement learning from human feedback''. In: arXiv preprint arXiv:2204.05862","author":"Bai Yuntao","year":"2022","unstructured":"Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, et al. ''Training a helpful and harmless assistant with reinforcement learning from human feedback''. In: arXiv preprint arXiv:2204.05862 (2022)."},{"key":"e_1_3_2_2_3_1","volume-title":"Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90%* ChatGPT Quality","author":"Chiang Wei-Lin","year":"2023","unstructured":"Wei-Lin Chiang et al. Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90%* ChatGPT Quality. Mar. 2023. url: https:\/\/lmsys.org\/blog\/2023-03-30-vicuna\/."},{"key":"e_1_3_2_2_4_1","volume-title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge''. In","author":"Clark Peter","year":"2018","unstructured":"Peter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord. ''Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge''. In: (2018). arXiv: 1803. 05457 [cs.AI]. url: https:\/\/arxiv.org\/abs\/1803.05457."},{"key":"e_1_3_2_2_5_1","unstructured":"DeepSeek-AI et al. DeepSeek-V3 Technical Report. 2025. arXiv: 2412. 19437 [cs.CL]. url: https:\/\/arxiv.org\/abs\/2412.19437."},{"key":"e_1_3_2_2_6_1","first-page":"30039","article-title":"A simulation framework for methods that learn from human feedback","volume":"36","author":"Dubois Yann","year":"2023","unstructured":"Yann Dubois, Chen Xuechen Li, Rohan Taori, Tianyi Zhang, Ishaan Gulrajani, Jimmy Ba, Carlos Guestrin, Percy S Liang, and Tatsunori B Hashimoto. ''Alpacafarm: A simulation framework for methods that learn from human feedback''. In: Advances in Neural Information Processing Systems 36 (2023), pp. 30039-30069.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.87.6.1200"},{"key":"e_1_3_2_2_8_1","volume-title":"The Llama 3 Herd of Models","author":"Aaron Grattafiori","year":"2024","unstructured":"Aaron Grattafiori et al. The Llama 3 Herd of Models. 2024. arXiv: 2407.21783 [cs.AI]. url: https:\/\/arxiv.org\/abs\/2407.21783."},{"key":"e_1_3_2_2_9_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR).","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. ''Measuring Massive Multitask Language Understanding''. In: Proceedings of the International Conference on Learning Representations (ICLR). 2021."},{"key":"e_1_3_2_2_10_1","volume-title":"GPT-4o System Card''. In: arXiv preprint arXiv:2410.21276","author":"Hurst Aaron","year":"2024","unstructured":"Aaron Hurst, Adam Lerer, Adam P Goucher, Adam Perelman, Aditya Ramesh, Aidan Clark, AJ Ostrow, Akila Welihinda, Alan Hayes, Alec Radford, et al. ''GPT-4o System Card''. In: arXiv preprint arXiv:2410.21276 (2024)."},{"key":"e_1_3_2_2_11_1","volume-title":"SWE-bench: Can Language Models Resolve Real-world Github Issues?'' In: The Twelfth International Conference on Learning Representations","author":"Jimenez Carlos E","year":"2024","unstructured":"Carlos E Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik R Narasimhan. ''SWE-bench: Can Language Models Resolve Real-world Github Issues?'' In: The Twelfth International Conference on Learning Representations. 2024."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.1124"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.92.6.1638"},{"key":"e_1_3_2_2_14_1","article-title":"Holistic Evaluation of Language Models","author":"Liang Percy","year":"2023","unstructured":"Percy Liang, Rishi Bommasani, Tony Lee, Dimitris Tsipras, Dilara Soylu, Michihiro Yasunaga, Yian Zhang, Deepak Narayanan, Yuhuai Wu, Ananya Kumar, et al. ''Holistic Evaluation of Language Models''. In: Transactions on Machine Learning Research (2023).","journal-title":"Transactions on Machine Learning Research ("},{"key":"e_1_3_2_2_15_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Liu Xiao","year":"2024","unstructured":"Xiao Liu, Hao Yu, Hanchen Zhang, Yifan Xu, Xuanyu Lei, Hanyu Lai, Yu Gu, Hangliang Ding, Kaiwen Men, Kejuan Yang, et al. ''AgentBench: Evaluating LLMs as Agents''. In: The Twelfth International Conference on Learning Representations. 2024."},{"key":"e_1_3_2_2_16_1","unstructured":"Mistral AI. Mistral Small 3. https:\/\/mistral.ai\/news\/mistral-small-3."},{"key":"e_1_3_2_2_17_1","unstructured":"OpenAI. GPT-4 Technical Report. 2023. arXiv: 2303.08774 [cs.CL]."},{"key":"e_1_3_2_2_18_1","volume-title":"GPT-4o System Card","author":"AI","year":"2024","unstructured":"OpenAI et al. GPT-4o System Card. 2024. arXiv: 2410. 21276 [cs.CL]. url: https:\/\/arxiv.org\/abs\/2410.21276."},{"key":"e_1_3_2_2_19_1","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, et al. ''Training language models to follow instructions with human feedback''. In: Advances in Neural Information Processing Systems 35 (2022), pp. 27730-27744.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_20_1","unstructured":"Sundar Pichai Demis Hassabis and Koray Kavukcuoglu. Introducing Gemini 2.0: our new AI model for the agentic era. https:\/\/blog.google\/technology\/googledeepmind\/ google-gemini-ai-update-december-2024\/."},{"key":"e_1_3_2_2_21_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Ruan Yangjun","year":"2024","unstructured":"Yangjun Ruan, Honghua Dong, Andrew Wang, Silviu Pitis, Yongchao Zhou, Jimmy Ba, Yann Dubois, Chris J Maddison, and Tatsunori Hashimoto. ''Identifying the Risks of LM Agents with an LM-Emulated Sandbox''. In: The Twelfth International Conference on Learning Representations. 2024."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.80.5.587"},{"key":"e_1_3_2_2_23_1","volume-title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","author":"Gemini Team","year":"2024","unstructured":"Gemini Team et al. Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context. 2024. arXiv: 2403. 05530 [cs.CL]. url: https:\/\/arxiv.org\/abs\/2403.05530."},{"key":"e_1_3_2_2_24_1","volume-title":"https:\/\/github.com\/clarice-wang\/openbench\/blob\/main\/Appendix.pdf","author":"Wang Clarice","year":"2025","unstructured":"Clarice Wang, Yimin Shi, and Xiaokui Xiao. Supplementary Appendix. https:\/\/github.com\/clarice-wang\/openbench\/blob\/main\/Appendix.pdf. 2025."},{"key":"e_1_3_2_2_25_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR).","author":"Wang Xingyao","year":"2024","unstructured":"Xingyao Wang, Zihan Wang, Jiateng Liu, Yangyi Chen, Lifan Yuan, Hao Peng, and Heng Ji. ''MINT: Evaluating LLMs in Multi-Turn Interaction with Tools and Language Feedback''. In: Proceedings of the International Conference on Learning Representations (ICLR). 2024."},{"key":"e_1_3_2_2_26_1","first-page":"24824","volume-title":"Denny Zhou, et al. ''Chain-of-thought prompting elicits reasoning in large language models''. In: Advances in neural information processing systems 35","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. ''Chain-of-thought prompting elicits reasoning in large language models''. In: Advances in neural information processing systems 35 (2022), pp. 24824-24837."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.0021-9029.2006.00055.x"},{"key":"e_1_3_2_2_28_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR).","author":"Xu Can","year":"2024","unstructured":"Can Xu, Qingfeng Sun, Kai Zheng, Xiubo Geng, Pu Zhao, Jiazhan Feng, Chongyang Tao, and Daxin Jiang. ''Wizardlm: Empowering large language models to follow complex instructions''. In: Proceedings of the International Conference on Learning Representations (ICLR). 2024."},{"key":"e_1_3_2_2_29_1","first-page":"20744","article-title":"Towards scalable real-world web interaction with grounded language agents","volume":"35","author":"Yao Shunyu","year":"2022","unstructured":"Shunyu Yao, Howard Chen, John Yang, and Karthik Narasimhan. ''Webshop: Towards scalable real-world web interaction with grounded language agents''. In: Advances in Neural Information Processing Systems 35 (2022), pp. 20744-20757.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_30_1","volume-title":"\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains","author":"Yao Shunyu","year":"2024","unstructured":"Shunyu Yao, Noah Shinn, Pedram Razavi, and Karthik Narasimhan. \u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains. 2024. arXiv: 2406.12045 [cs.AI]. url: https:\/\/arxiv.org\/abs\/2406.12045."},{"key":"e_1_3_2_2_31_1","volume-title":"Proceedings of the NeurIPS 2023 Datasets and Benchmarks Track.","author":"Lianmin","year":"2023","unstructured":"Lianmin Zheng et al. ''Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena''. In: Proceedings of the NeurIPS 2023 Datasets and Benchmarks Track. 2023."},{"key":"e_1_3_2_2_32_1","first-page":"55006","article-title":"Lima: Less is more for alignment","volume":"36","author":"Zhou Chunting","year":"2023","unstructured":"Chunting Zhou, Pengfei Liu, Puxin Xu, Srinivasan Iyer, Jiao Sun, Yuning Mao, Xuezhe Ma, Avia Efrat, Ping Yu, Lili Yu, et al. ''Lima: Less is more for alignment''. In: Advances in Neural Information Processing Systems 36 (2023), pp. 55006-55021.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_33_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Zhou Shuyan","year":"2024","unstructured":"Shuyan Zhou, Frank F Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, et al. ''WebArena: A Realistic Web Environment for Building Autonomous Agents''. In: The Twelfth International Conference on Learning Representations. 2024."}],"event":{"name":"KDD '25: The 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Toronto ON Canada","acronym":"KDD '25","sponsor":["SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3711896.3737390","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T17:59:28Z","timestamp":1777571968000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3711896.3737390"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,3]]},"references-count":33,"alternative-id":["10.1145\/3711896.3737390","10.1145\/3711896"],"URL":"https:\/\/doi.org\/10.1145\/3711896.3737390","relation":{},"subject":[],"published":{"date-parts":[[2025,8,3]]},"assertion":[{"value":"2025-08-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}