{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T15:20:41Z","timestamp":1783437641763,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","funder":[{"name":"Natural Science Foundation of China","award":["U21B2026, 62372260"],"award-info":[{"award-number":["U21B2026, 62372260"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,10]]},"DOI":"10.1145\/3746252.3761391","type":"proceedings-article","created":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T00:29:28Z","timestamp":1762561768000},"page":"3952-3962","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["StepTool: Enhancing Multi-Step Tool Usage in LLMs via Step-Grained Reinforcement Learning"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2942-1576","authenticated-orcid":false,"given":"Yuanqing","family":"Yu","sequence":"first","affiliation":[{"name":"DCST, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7176-8234","authenticated-orcid":false,"given":"Zhefan","family":"Wang","sequence":"additional","affiliation":[{"name":"DCST, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5604-7527","authenticated-orcid":false,"given":"Weizhi","family":"Ma","sequence":"additional","affiliation":[{"name":"Institute for AI Industry Research, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6297-9085","authenticated-orcid":false,"given":"Shuai","family":"Wang","sequence":"additional","affiliation":[{"name":"Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5730-8792","authenticated-orcid":false,"given":"Chuhan","family":"Wu","sequence":"additional","affiliation":[{"name":"Huawei Noah's Ark Lab, Huawei Noah's Ark Lab, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9393-4854","authenticated-orcid":false,"given":"Zhiqiang","family":"Guo","sequence":"additional","affiliation":[{"name":"DCST, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3158-1920","authenticated-orcid":false,"given":"Min","family":"Zhang","sequence":"additional","affiliation":[{"name":"DCST, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,11,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Ibrahim Abdelaziz Kinjal Basu Mayank Agarwal Sadhana Kumaravel Matthew Stallone Rameswar Panda Yara Rizk GP Bhargav Maxwell Crouse Chulaka Gunasekara et al. 2024. Granite-function calling model: Introducing function calling abilities via multi-task learning of granular tasks. arXiv preprint arXiv:2407.00121 (2024).","DOI":"10.18653\/v1\/2024.emnlp-industry.85"},{"key":"e_1_3_2_1_2_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language models are few-shot learners. Advances in neural information processing systems Vol. 33 (2020) 1877-1901."},{"key":"e_1_3_2_1_3_1","volume-title":"International Conference on Machine Learning. PMLR, 3676-3713","author":"Carta Thomas","year":"2023","unstructured":"Thomas Carta, Cl\u00e9ment Romac, Thomas Wolf, Sylvain Lamprier, Olivier Sigaud, and Pierre-Yves Oudeyer. 2023. Grounding large language models in interactive environments with online reinforcement learning. In International Conference on Machine Learning. PMLR, 3676-3713."},{"key":"e_1_3_2_1_4_1","volume-title":"Advancing Tool-Augmented Large Language Models: Integrating Insights from Errors in Inference Trees. arXiv preprint arXiv:2406.07115","author":"Chen Sijia","year":"2024","unstructured":"Sijia Chen, Yibo Wang, Yi-Feng Wu, Qing-Guo Chen, Zhao Xu, Weihua Luo, Kaifu Zhang, and Lijun Zhang. 2024. Advancing Tool-Augmented Large Language Models: Integrating Insights from Errors in Inference Trees. arXiv preprint arXiv:2406.07115 (2024)."},{"key":"e_1_3_2_1_5_1","volume-title":"Cohen","author":"Chen Wenhu","year":"2023","unstructured":"Wenhu Chen, Xueguang Ma, Xinyi Wang, and William W. Cohen. 2023. Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks. Transactions on Machine Learning Research (2023)."},{"key":"e_1_3_2_1_6_1","volume-title":"Deep reinforcement learning from human preferences. Advances in neural information processing systems","author":"Christiano Paul F","year":"2017","unstructured":"Paul F Christiano, Jan Leike, Tom Brown, Miljan Martic, Shane Legg, and Dario Amodei. 2017. Deep reinforcement learning from human preferences. Advances in neural information processing systems, Vol. 30 (2017)."},{"key":"e_1_3_2_1_7_1","volume-title":"StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models. arXiv preprint arXiv:2403.07714","author":"Guo Zhicheng","year":"2024","unstructured":"Zhicheng Guo, Sijie Cheng, Hao Wang, Shihao Liang, Yujia Qin, Peng Li, Zhiyuan Liu, Maosong Sun, and Yang Liu. 2024. StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models. arXiv preprint arXiv:2403.07714 (2024)."},{"key":"e_1_3_2_1_8_1","volume-title":"Christoforos Nalmpantis, Jane Dwivedi-Yu, Maksym Zhuravinskyi, Eric Hambro, Sainbayar Sukhbaatar, and Roberta Raileanu.","author":"Havrilla Alex","year":"2024","unstructured":"Alex Havrilla, Yuqing Du, Sharath Chandra Raparthy, Christoforos Nalmpantis, Jane Dwivedi-Yu, Maksym Zhuravinskyi, Eric Hambro, Sainbayar Sukhbaatar, and Roberta Raileanu. 2024. Teaching large language models to reason with reinforcement learning. arXiv preprint arXiv:2403.04642 (2024)."},{"key":"e_1_3_2_1_9_1","volume-title":"Step-dpo: Step-wise preference optimization for long-chain reasoning of llms. arXiv preprint arXiv:2406.18629","author":"Lai Xin","year":"2024","unstructured":"Xin Lai, Zhuotao Tian, Yukang Chen, Senqiao Yang, Xiangru Peng, and Jiaya Jia. 2024. Step-dpo: Step-wise preference optimization for long-chain reasoning of llms. arXiv preprint arXiv:2406.18629 (2024)."},{"key":"e_1_3_2_1_10_1","volume-title":"Let's Verify Step by Step. arXiv preprint arXiv:2305.20050","author":"Lightman Hunter","year":"2023","unstructured":"Hunter Lightman, Vineet Kosaraju, Yura Burda, Harri Edwards, Bowen Baker, Teddy Lee, Jan Leike, John Schulman, Ilya Sutskever, and Karl Cobbe. 2023. Let's Verify Step by Step. arXiv preprint arXiv:2305.20050 (2023)."},{"key":"e_1_3_2_1_11_1","unstructured":"Xukun Liu Zhiyuan Peng Xiaoyuan Yi Xing Xie Lirong Xiang Yuchen Liu and Dongkuan Xu. 2024b. ToolNet: Connecting Large Language Models with Massive Tools via Tool Graph. arXiv:2403.00839 [cs.AI]"},{"key":"e_1_3_2_1_12_1","unstructured":"Yanming Liu Xinyue Peng Yuwei Zhang Jiannan Cao Xuhong Zhang Sheng Cheng Xun Wang Jianwei Yin and Tianyu Du. 2024c. Tool-Planner: Dynamic Solution Tree Planning for Large Language Model with Tool Clustering. arXiv:2406.03807 [cs.AI]"},{"key":"e_1_3_2_1_13_1","volume-title":"Apigen: Automated pipeline for generating verifiable and diverse function-calling datasets. arXiv preprint arXiv:2406.18518","author":"Liu Zuxin","year":"2024","unstructured":"Zuxin Liu, Thai Hoang, Jianguo Zhang, Ming Zhu, Tian Lan, Shirley Kokane, Juntao Tan, Weiran Yao, Zhiwei Liu, Yihao Feng, et al., 2024a. Apigen: Automated pipeline for generating verifiable and diverse function-calling datasets. arXiv preprint arXiv:2406.18518 (2024)."},{"key":"e_1_3_2_1_14_1","volume-title":"Let's reward step by step: Step-Level reward model as the Navigators for Reasoning. arXiv preprint arXiv:2310.10080","author":"Ma Qianli","year":"2023","unstructured":"Qianli Ma, Haotian Zhou, Tingkai Liu, Jianbo Yuan, Pengfei Liu, Yang You, and Hongxia Yang. 2023. Let's reward step by step: Step-Level reward model as the Navigators for Reasoning. arXiv preprint arXiv:2310.10080 (2023)."},{"key":"e_1_3_2_1_15_1","volume-title":"Chenglong Wang, Oleksandr Polozov, Christopher Meek, Dragomir Radev, and Jianfeng Gao.","author":"Ni Ansong","year":"2022","unstructured":"Ansong Ni, Jeevana Priya Inala, Chenglong Wang, Oleksandr Polozov, Christopher Meek, Dragomir Radev, and Jianfeng Gao. 2022. Learning math reasoning from self-sampled correct and partially-correct solutions. arXiv preprint arXiv:2205.14318 (2022)."},{"key":"e_1_3_2_1_16_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arXiv preprint arXiv:2303.08774 (2023)."},{"key":"e_1_3_2_1_17_1","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray et al. 2022. Training language models to follow instructions with human feedback. Advances in neural information processing systems Vol. 35 (2022) 27730-27744."},{"key":"e_1_3_2_1_18_1","volume-title":"Gorilla: Large language model connected with massive apis. arXiv preprint arXiv:2305.15334","author":"Patil Shishir G","year":"2023","unstructured":"Shishir G Patil, Tianjun Zhang, Xin Wang, and Joseph E Gonzalez. 2023. Gorilla: Large language model connected with massive apis. arXiv preprint arXiv:2305.15334 (2023)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Yujia Qin Shengding Hu Yankai Lin Weize Chen Ning Ding Ganqu Cui Zheni Zeng Yufei Huang Chaojun Xiao Chi Han Yi Ren Fung Yusheng Su Huadong Wang Cheng Qian Runchu Tian Kunlun Zhu Shihao Liang Xingyu Shen Bokai Xu Zhen Zhang Yining Ye Bowen Li Ziwei Tang Jing Yi Yuzhang Zhu Zhenning Dai Lan Yan Xin Cong Yaxi Lu Weilin Zhao Yuxiang Huang Junxi Yan Xu Han Xian Sun Dahai Li Jason Phang Cheng Yang Tongshuang Wu Heng Ji Zhiyuan Liu and Maosong Sun. 2024. Tool Learning with Foundation Models. arXiv:2304.08354 [cs.CL]","DOI":"10.1145\/3704435"},{"key":"e_1_3_2_1_20_1","volume-title":"Toolllm: Facilitating large language models to master 16000 real-world apis. arXiv preprint arXiv:2307.16789","author":"Qin Yujia","year":"2023","unstructured":"Yujia Qin, Shihao Liang, Yining Ye, Kunlun Zhu, Lan Yan, Yaxi Lu, Yankai Lin, Xin Cong, Xiangru Tang, Bill Qian, et al., 2023. Toolllm: Facilitating large language models to master 16000 real-world apis. arXiv preprint arXiv:2307.16789 (2023)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3627673.3679847"},{"key":"e_1_3_2_1_22_1","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Rafailov Rafael","year":"2024","unstructured":"Rafael Rafailov, Archit Sharma, Eric Mitchell, Christopher D Manning, Stefano Ermon, and Chelsea Finn. 2024. Direct preference optimization: Your language model is secretly a reward model. Advances in Neural Information Processing Systems, Vol. 36 (2024)."},{"key":"e_1_3_2_1_23_1","volume-title":"Maria Lomeli, Eric Hambro, Luke Zettlemoyer, Nicola Cancedda, and Thomas Scialom.","author":"Schick Timo","year":"2024","unstructured":"Timo Schick, Jane Dwivedi-Yu, Roberto Dess`i, Roberta Raileanu, Maria Lomeli, Eric Hambro, Luke Zettlemoyer, Nicola Cancedda, and Thomas Scialom. 2024. Toolformer: Language models can teach themselves to use tools. Advances in Neural Information Processing Systems, Vol. 36 (2024)."},{"key":"e_1_3_2_1_24_1","volume-title":"Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347","author":"Schulman John","year":"2017","unstructured":"John Schulman, Filip Wolski, Prafulla Dhariwal, Alec Radford, and Oleg Klimov. 2017. Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347 (2017)."},{"key":"e_1_3_2_1_25_1","volume-title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models. arXiv:2402.03300","author":"Shao Zhihong","year":"2024","unstructured":"Zhihong Shao, Peiyi Wang, Qihao Zhu, Runxin Xu, Junxiao Song, Mingchuan Zhang, YK Li, Y Wu, and Daya Guo. 2024. Deepseekmath: Pushing the limits of mathematical reasoning in open language models. arXiv:2402.03300 (2024)."},{"key":"e_1_3_2_1_26_1","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Shen Yongliang","year":"2024","unstructured":"Yongliang Shen, Kaitao Song, Xu Tan, Dongsheng Li, Weiming Lu, and Yueting Zhuang. 2024. Hugginggpt: Solving ai tasks with chatgpt and its friends in hugging face. Advances in Neural Information Processing Systems, Vol. 36 (2024)."},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR).","author":"Shridhar Mohit","year":"2021","unstructured":"Mohit Shridhar, Xingdi Yuan, Marc-Alexandre C\u00f4t\u00e9, Yonatan Bisk, Adam Trischler, and Matthew Hausknecht. 2021. ALFWorld: Aligning Text and Embodied Environments for Interactive Learning. In Proceedings of the International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.409"},{"key":"e_1_3_2_1_29_1","volume-title":"Policy gradient methods for reinforcement learning with function approximation. Advances in neural information processing systems","author":"Sutton Richard S","year":"1999","unstructured":"Richard S Sutton, David McAllester, Satinder Singh, and Yishay Mansour. 1999. Policy gradient methods for reinforcement learning with function approximation. Advances in neural information processing systems, Vol. 12 (1999)."},{"key":"e_1_3_2_1_30_1","volume-title":"True knowledge comes from practice: Aligning llms with embodied environments via reinforcement learning. arXiv preprint arXiv:2401.14151","author":"Tan Weihao","year":"2024","unstructured":"Weihao Tan, Wentao Zhang, Shanqi Liu, Longtao Zheng, Xinrun Wang, and Bo An. 2024. True knowledge comes from practice: Aligning llms with embodied environments via reinforcement learning. arXiv preprint arXiv:2401.14151 (2024)."},{"key":"e_1_3_2_1_31_1","volume-title":"Toolalpaca: Generalized tool learning for language models with 3000 simulated cases. arXiv preprint arXiv:2306.05301","author":"Tang Qiaoyu","year":"2023","unstructured":"Qiaoyu Tang, Ziliang Deng, Hongyu Lin, Xianpei Han, Qiao Liang, Boxi Cao, and Le Sun. 2023. Toolalpaca: Generalized tool learning for language models with 3000 simulated cases. arXiv preprint arXiv:2306.05301 (2023)."},{"key":"e_1_3_2_1_32_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale et al. 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_33_1","volume-title":"Solving math word problems with process-and outcome-based feedback. arXiv preprint arXiv:2211.14275","author":"Uesato Jonathan","year":"2022","unstructured":"Jonathan Uesato, Nate Kushman, Ramana Kumar, Francis Song, Noah Siegel, Lisa Wang, Antonia Creswell, Geoffrey Irving, and Irina Higgins. 2022. Solving math word problems with process-and outcome-based feedback. arXiv preprint arXiv:2211.14275 (2022)."},{"key":"e_1_3_2_1_34_1","volume-title":"Benjamin Van Durme, and Yu Su","author":"Wang Boshi","year":"2024","unstructured":"Boshi Wang, Hao Fang, Jason Eisner, Benjamin Van Durme, and Yu Su. 2024. LLMs in the Imaginarium: Tool Learning through Simulated Trial and Error. arXiv:2403.04746 [cs.CL]"},{"key":"e_1_3_2_1_35_1","volume-title":"Math-shepherd: Verify and reinforce llms step-by-step without human annotations. CoRR, abs\/2312.08935","author":"Wang Peiyi","year":"2023","unstructured":"Peiyi Wang, Lei Li, Zhihong Shao, RX Xu, Damai Dai, Yifei Li, Deli Chen, Y Wu, and Zhifang Sui. 2023. Math-shepherd: Verify and reinforce llms step-by-step without human annotations. CoRR, abs\/2312.08935 (2023)."},{"key":"e_1_3_2_1_36_1","volume-title":"Denny Zhou, et al.","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al., 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in neural information processing systems, Vol. 35 (2022), 24824-24837."},{"key":"e_1_3_2_1_37_1","volume-title":"Reinforcing Language Agents via Policy Optimization with Action Decomposition. arXiv preprint arXiv:2405.15821","author":"Wen Muning","year":"2024","unstructured":"Muning Wen, Ziyu Wan, Weinan Zhang, Jun Wang, and Ying Wen. 2024. Reinforcing Language Agents via Policy Optimization with Action Decomposition. arXiv preprint arXiv:2405.15821 (2024)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Ronald J Williams. 1992. Simple statistical gradient-following algorithms for connectionist reinforcement learning. Machine learning Vol. 8 (1992) 229-256.","DOI":"10.1023\/A:1022672621406"},{"key":"e_1_3_2_1_39_1","unstructured":"An Yang Baosong Yang Binyuan Hui Bo Zheng Bowen Yu Chang Zhou Chengpeng Li Chengyuan Li Dayiheng Liu Fei Huang et al. 2024. Qwen2 technical report. arXiv preprint arXiv:2407.10671 (2024)."},{"key":"e_1_3_2_1_40_1","volume-title":"Scaling relationship on learning mathematical reasoning with large language models. arXiv preprint arXiv:2308.01825","author":"Yuan Zheng","year":"2023","unstructured":"Zheng Yuan, Hongyi Yuan, Chengpeng Li, Guanting Dong, Keming Lu, Chuanqi Tan, Chang Zhou, and Jingren Zhou. 2023. Scaling relationship on learning mathematical reasoning with large language models. arXiv preprint arXiv:2308.01825 (2023)."},{"key":"e_1_3_2_1_41_1","unstructured":"Aohan Zeng Xiao Liu Zhengxiao Du Zihan Wang Hanyu Lai Ming Ding Zhuoyi Yang Yifan Xu Wendi Zheng Xiao Xia et al. 2022. Glm-130b: An open bilingual pre-trained model. arXiv preprint arXiv:2210.02414 (2022)."},{"key":"e_1_3_2_1_42_1","unstructured":"Yuanhang Zheng Peng Li Wei Liu Yang Liu Jian Luan and Bin Wang. 2024. ToolRerank: Adaptive and Hierarchy-Aware Reranking for Tool Retrieval. arXiv:2403.06551 [cs.IR]"},{"key":"e_1_3_2_1_43_1","volume-title":"Archer: Training language model agents via hierarchical multi-turn rl. arXiv preprint arXiv:2402.19446","author":"Zhou Yifei","year":"2024","unstructured":"Yifei Zhou, Andrea Zanette, Jiayi Pan, Sergey Levine, and Aviral Kumar. 2024. Archer: Training language model agents via hierarchical multi-turn rl. arXiv preprint arXiv:2402.19446 (2024)."}],"event":{"name":"CIKM '25: The 34th ACM International Conference on Information and Knowledge Management","location":"Seoul Republic of Korea","acronym":"CIKM '25","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the 34th ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3746252.3761391","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T01:53:12Z","timestamp":1765504392000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3746252.3761391"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,10]]},"references-count":43,"alternative-id":["10.1145\/3746252.3761391","10.1145\/3746252"],"URL":"https:\/\/doi.org\/10.1145\/3746252.3761391","relation":{},"subject":[],"published":{"date-parts":[[2025,11,10]]},"assertion":[{"value":"2025-11-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}