{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T04:07:43Z","timestamp":1779422863894,"version":"3.53.1"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,5,26]],"date-time":"2026-05-26T00:00:00Z","timestamp":1779753600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["IIS-2142827"],"award-info":[{"award-number":["IIS-2142827"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["IIS-2234058"],"award-info":[{"award-number":["IIS-2234058"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,5,26]]},"DOI":"10.1145\/3786335.3813164","type":"proceedings-article","created":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:16:22Z","timestamp":1779419782000},"page":"197-213","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Composing Policy Gradients and Prompt Optimization for Language Model Programs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-1002-1959","authenticated-orcid":false,"given":"Noah","family":"Ziems","sequence":"first","affiliation":[{"name":"MIT, Cambridge, USA and University of Notre Dame, South Bend, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-6265-2055","authenticated-orcid":false,"given":"Dilara","family":"Soylu","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0409-8212","authenticated-orcid":false,"given":"Lakshya A","family":"Agrawal","sequence":"additional","affiliation":[{"name":"UC Berkeley, Berkeley, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-5978-4901","authenticated-orcid":false,"given":"Isaac","family":"Miller","sequence":"additional","affiliation":[{"name":"Anyscale, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-7429-8062","authenticated-orcid":false,"given":"Liheng","family":"Lai","sequence":"additional","affiliation":[{"name":"UC Berkeley, Berkeley, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1481-5069","authenticated-orcid":false,"given":"Chen","family":"Qian","sequence":"additional","affiliation":[{"name":"CMU, Pittsburgh, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-2007-2596","authenticated-orcid":false,"given":"Kaiqiang","family":"Song","sequence":"additional","affiliation":[{"name":"Zoom, Inc., Bellevue, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3009-519X","authenticated-orcid":false,"given":"Meng","family":"Jiang","sequence":"additional","affiliation":[{"name":"University of Notre Dame, South Bend, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5983-2904","authenticated-orcid":false,"given":"Dan","family":"Klein","sequence":"additional","affiliation":[{"name":"UC Berkeley, Berkeley, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7547-7204","authenticated-orcid":false,"given":"Matei","family":"Zaharia","sequence":"additional","affiliation":[{"name":"UC Berkeley, Berkeley, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1695-1014","authenticated-orcid":false,"given":"Karel","family":"D\u2019Oosterlinck","sequence":"additional","affiliation":[{"name":"Contextual AI, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7978-6055","authenticated-orcid":false,"given":"Christopher","family":"Potts","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9836-4413","authenticated-orcid":false,"given":"Omar","family":"Khattab","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,5,26]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"Lakshya\u00a0A Agrawal Shangyin Tan Dilara Soylu Noah Ziems Rishi Khare Krista Opsahl-Ong Arnav Singhvi Herumb Shandilya Michael\u00a0J Ryan Meng Jiang Christopher Potts Koushik Sen Alexandros\u00a0G. Dimakis Ion Stoica Dan Klein Matei Zaharia and Omar Khattab. 2026. GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2507.19457\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2507.19457"},{"key":"e_1_3_3_1_3_2","unstructured":"William Brown. 2025. Verifiers: Reinforcement Learning with LLMs in Verifiable Environments. https:\/\/github.com\/willccbb\/verifiers."},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.nlp4convai-1.5"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.222"},{"key":"e_1_3_3_1_6_2","first-page":"13481","volume-title":"Proceedings of the 41st International Conference on Machine Learning","author":"Fernando Chrisantha","year":"2024","unstructured":"Chrisantha Fernando, Dylan Banarse, Henryk Michalewski, Simon Osindero, and Tim Rockt\u00e4schel. 2024. Promptbreeder: self-referential self-improvement via prompt evolution. In Proceedings of the 41st International Conference on Machine Learning. 13481\u201313544."},{"key":"e_1_3_3_1_7_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan et\u00a0al. 2024. The llama 3 herd of models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.21783 (2024)."},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"crossref","unstructured":"Yaru Hao Zewen Chi Li Dong and Furu Wei. 2023. Optimizing prompts for text-to-image generation. Advances in Neural Information Processing Systems 36 (2023) 66923\u201366939.","DOI":"10.52202\/075280-2923"},{"key":"e_1_3_3_1_9_2","unstructured":"Edward\u00a0J. Hu Yelong Shen Phillip Wallis Zeyuan Allen-Zhu Yuanzhi Li Shean Wang Lu Wang and Weizhu Chen. 2021. LoRA: Low-Rank Adaptation of Large Language Models. ArXiv preprint abs\/2106.09685 (2021). https:\/\/arxiv.org\/abs\/2106.09685"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.309"},{"key":"e_1_3_3_1_11_2","unstructured":"Bowen Jin Hansi Zeng Zhenrui Yue Jinsung Yoon Sercan Arik Dong Wang Hamed Zamani and Jiawei Han. 2025. Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning. ArXiv preprint abs\/2503.09516 (2025). https:\/\/arxiv.org\/abs\/2503.09516"},{"key":"e_1_3_3_1_12_2","volume-title":"ICLRThe Twelfth International Conference on Learning Representations","author":"Khattab Omar","year":"2024","unstructured":"Omar Khattab, Arnav Singhvi, Paridhi Maheshwari, Zhiyuan Zhang, Keshav Santhanam, Sri Vardhamanan, Saiful Haq, Ashutosh Sharma, Thomas\u00a0T. Joshi, Hanna Moazam, Heather Miller, Matei Zaharia, and Christopher Potts. 2024. DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines, In ICLR. The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_3_1_14_2","unstructured":"Yoonho Lee Roshen Nair Qizheng Zhang Kangwook Lee Omar Khattab and Chelsea Finn. 2026. Meta-Harness: End-to-End Optimization of Model Harnesses. arxiv:https:\/\/arXiv.org\/abs\/2603.28052\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2603.28052"},{"key":"e_1_3_3_1_15_2","unstructured":"MetaAI. 2024. Meta Llama 3.1-8B-Instruct: Generation Configuration. https:\/\/huggingface.co\/meta-llama\/Llama-3.1-8B-Instruct\/blob\/main\/generation_config.json. Accessed: 2025-07-29."},{"key":"e_1_3_3_1_16_2","unstructured":"MiniMax : Aili Chen Aonian Li Bangwei Gong Binyang Jiang Bo Fei Bo Yang Boji Shan Changqing Yu Chao Wang Cheng Zhu Chengjun Xiao Chengyu Du Chi Zhang Chu Qiao Chunhao Zhang Chunhui Du Congchao Guo Da Chen Deming Ding Dianjun Sun Dong Li Enwei Jiao Haigang Zhou Haimo Zhang Han Ding Haohai Sun Haoyu Feng Huaiguang Cai Haichao Zhu Jian Sun Jiaqi Zhuang Jiaren Cai Jiayuan Song Jin Zhu Jingyang Li Jinhao Tian Jinli Liu Junhao Xu Junjie Yan Junteng Liu Junxian He Kaiyi Feng Ke Yang Kecheng Xiao Le Han Leyang Wang Lianfei Yu Liheng Feng Lin Li Lin Zheng Linge Du Lingyu Yang Lunbin Zeng Minghui Yu Mingliang Tao Mingyuan Chi Mozhi Zhang Mujie Lin Nan Hu Nongyu Di Peng Gao Pengfei Li Pengyu Zhao Qibing Ren Qidi Xu Qile Li Qin Wang Rong Tian Ruitao Leng Shaoxiang Chen Shaoyu Chen Shengmin Shi Shitong Weng Shuchang Guan Shuqi Yu Sichen Li Songquan Zhu Tengfei Li Tianchi Cai Tianrun Liang Weiyu Cheng Weize Kong Wenkai Li Xiancai Chen Xiangjun Song Xiao Luo Xiao Su Xiaobo Li Xiaodong Han Xinzhu Hou Xuan Lu Xun Zou Xuyang Shen Yan Gong Yan Ma Yang Wang Yiqi Shi Yiran Zhong Yonghong Duan Yongxiang Fu Yongyi Hu Yu Gao Yuanxiang Fan Yufeng Yang Yuhao Li Yulin Hu Yunan Huang Yunji Li Yunzhi Xu Yuxin Mao Yuxuan Shi Yuze Wenren Zehan Li Zelin Li Zhanxu Tian Zhengmao Zhu Zhenhua Fan Zhenzhen Wu Zhichao Xu Zhihang Yu Zhiheng Lyu Zhuo Jiang Zibo Gao Zijia Wu Zijian Song and Zijun Sun. 2025. MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention. arxiv:https:\/\/arXiv.org\/abs\/2506.13585\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2506.13585"},{"key":"e_1_3_3_1_17_2","unstructured":"Reiichiro Nakano Jacob Hilton Suchir Balaji Jeff Wu Long Ouyang Christina Kim Christopher Hesse Shantanu Jain Vineet Kosaraju William Saunders Xu Jiang Karl Cobbe Tyna Eloundou Gretchen Krueger Kevin Button Matthew Knight Benjamin Chess and John Schulman. 2021. WebGPT: Browser-assisted question-answering with human feedback. CoRR abs\/2112.09332 (2021). arXiv:https:\/\/arXiv.org\/abs\/2112.09332https:\/\/arxiv.org\/abs\/2112.09332"},{"key":"e_1_3_3_1_18_2","unstructured":"OpenAI. 2025. GPT-4.1-mini. https:\/\/platform.openai.com\/docs\/models\/gpt-4.1-mini. Accessed: 2025-07-17."},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.525"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray et\u00a0al. 2022. Training language models to follow instructions with human feedback. Advances in neural information processing systems 35 (2022) 27730\u201327744.","DOI":"10.52202\/068431-2011"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.494"},{"key":"e_1_3_3_1_22_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.292"},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"crossref","unstructured":"Rafael Rafailov Archit Sharma Eric Mitchell Christopher\u00a0D Manning Stefano Ermon and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. Advances in neural information processing systems 36 (2023) 53728\u201353741.","DOI":"10.52202\/075280-2338"},{"key":"e_1_3_3_1_24_2","unstructured":"Keshav Santhanam Omar Khattab Jon Saad-Falcon Christopher Potts and Matei Zaharia. 2021. ColBERTv2: Effective and Efficient Retrieval via Lightweight Late Interaction. ArXiv preprint abs\/2112.01488 (2021). https:\/\/arxiv.org\/abs\/2112.01488"},{"key":"e_1_3_3_1_25_2","unstructured":"John Schulman Filip Wolski Prafulla Dhariwal Alec Radford and Oleg Klimov. 2017. Proximal policy optimization algorithms. ArXiv preprint abs\/1707.06347 (2017). https:\/\/arxiv.org\/abs\/1707.06347"},{"key":"e_1_3_3_1_26_2","unstructured":"Zhihong Shao Peiyi Wang Qihao Zhu Runxin Xu Junxiao Song Xiao Bi Haowei Zhang Mingchuan Zhang Y.\u00a0K. Li Y. Wu and Daya Guo. 2024. DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models. ArXiv preprint abs\/2402.03300 (2024). https:\/\/arxiv.org\/abs\/2402.03300"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.346"},{"key":"e_1_3_3_1_28_2","unstructured":"Li Siyan Vethavikashini\u00a0Chithrra Raghuram Omar Khattab Julia Hirschberg and Zhou Yu. 2024. PAPILLON: PrivAcy Preservation from Internet-based and Local Language MOdel ENsembles. ArXiv preprint abs\/2410.17127 (2024). https:\/\/arxiv.org\/abs\/2410.17127"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.597"},{"key":"e_1_3_3_1_30_2","unstructured":"Shangyin Tan Lakshya\u00a0A Agrawal Arnav Singhvi Liheng Lai Michael\u00a0J Ryan Dan Klein Omar Khattab Koushik Sen and Matei Zaharia. 2025. LangProBe: a Language Programs Benchmark. ArXiv preprint abs\/2502.20315 (2025). https:\/\/arxiv.org\/abs\/2502.20315"},{"key":"e_1_3_3_1_31_2","volume-title":"TRL: Transformers Reinforcement Learning","author":"Werra Leandro von","year":"2020","unstructured":"Leandro von Werra, Younes Belkada, Lewis Tunstall, Edward Beeching, Tristan Thrush, Nathan Lambert, Shengyi Huang, Kashif Rasul, and Quentin Gallou\u00e9dec. 2020. TRL: Transformers Reinforcement Learning. https:\/\/github.com\/huggingface\/trl"},{"key":"e_1_3_3_1_32_2","unstructured":"Zihan Wang Kangrui Wang Qineng Wang Pingyue Zhang Linjie Li Zhengyuan Yang Xing Jin Kefan Yu Minh\u00a0Nhat Nguyen Licheng Liu Eli Gottlieb Yiping Lu Kyunghyun Cho Jiajun Wu Li Fei-Fei Lijuan Wang Yejin Choi and Manling Li. 2025. RAGEN: Understanding Self-Evolution in LLM Agents via Multi-Turn Reinforcement Learning. ArXiv preprint abs\/2504.20073 (2025). https:\/\/arxiv.org\/abs\/2504.20073"},{"key":"e_1_3_3_1_33_2","unstructured":"Jason Wei Xuezhi Wang Dale Schuurmans Maarten Bosma Brian Ichter Fei Xia Ed Chi Quoc Le and Denny Zhou. 2022. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. ArXiv preprint abs\/2201.11903 (2022). https:\/\/arxiv.org\/abs\/2201.11903"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"crossref","unstructured":"Yuxin Wen Neel Jain John Kirchenbauer Micah Goldblum Jonas Geiping and Tom Goldstein. 2023. Hard prompts made easy: Gradient-based discrete optimization for prompt tuning and discovery. Advances in Neural Information Processing Systems 36 (2023) 51008\u201351025.","DOI":"10.52202\/075280-2219"},{"key":"e_1_3_3_1_35_2","unstructured":"Yixuan\u00a0Even Xu Yash Savani Fei Fang and Zico Kolter. 2025. Not All Rollouts are Useful: Down-Sampling Rollouts in LLM Reinforcement Learning. ArXiv preprint abs\/2504.13818 (2025). https:\/\/arxiv.org\/abs\/2504.13818"},{"key":"e_1_3_3_1_36_2","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv Chujie Zheng Dayiheng Liu Fan Zhou Fei Huang Feng Hu Hao Ge Haoran Wei Huan Lin Jialong Tang Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jing Zhou Jingren Zhou Junyang Lin Kai Dang Keqin Bao Kexin Yang Le Yu Lianghao Deng Mei Li Mingfeng Xue Mingze Li Pei Zhang Peng Wang Qin Zhu Rui Men Ruize Gao Shixuan Liu Shuang Luo Tianhao Li Tianyi Tang Wenbiao Yin Xingzhang Ren Xinyu Wang Xinyu Zhang Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yinger Zhang Yu Wan Yuqiong Liu Zekun Wang Zeyu Cui Zhenru Zhang Zhipeng Zhou and Zihan Qiu. 2025. Qwen3 Technical Report. ArXiv preprint abs\/2505.09388 (2025). https:\/\/arxiv.org\/abs\/2505.09388"},{"key":"e_1_3_3_1_37_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Yang Chengrun","year":"2024","unstructured":"Chengrun Yang, Xuezhi Wang, Yifeng Lu, Hanxiao Liu, Quoc\u00a0V Le, Denny Zhou, and Xinyun Chen. 2024. Large Language Models as Optimizers. In The Twelfth International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=Bb4VGOWELI"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1259"},{"key":"e_1_3_3_1_39_2","unstructured":"Siliang Zeng Quan Wei William Brown Oana Frunza Yuriy Nevmyvaka and Mingyi Hong. 2025. Reinforcing Multi-Turn Reasoning in LLM Agents via Turn-Level Credit Assignment. ArXiv preprint abs\/2505.11821 (2025). https:\/\/arxiv.org\/abs\/2505.11821"},{"key":"e_1_3_3_1_40_2","volume-title":"The Eleventh International Conference on Learning Representations","author":"Zhang Tianjun","year":"2023","unstructured":"Tianjun Zhang, Xuezhi Wang, Denny Zhou, Dale Schuurmans, and Joseph\u00a0E. Gonzalez. 2023. TEMPERA: Test-Time Prompt Editing via Reinforcement Learning. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=gSHyqBijPFO"},{"key":"e_1_3_3_1_41_2","volume-title":"The Eleventh International Conference on Learning Representations","author":"Zhou Yongchao","year":"2023","unstructured":"Yongchao Zhou, Andrei\u00a0Ioan Muresanu, Ziwen Han, Keiran Paster, Silviu Pitis, Harris Chan, and Jimmy Ba. 2023. Large Language Models are Human-Level Prompt Engineers. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=92gvk82DE-"}],"event":{"name":"CAIS '26: ACM Conference on AI and Agentic Systems","location":"San Jose CA USA","acronym":"CAIS '26"},"container-title":["Proceedings of the ACM Conference on AI and Agentic Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3786335.3813164","content-type":"text\/html","content-version":"vor","intended-application":"syndication"}],"deposited":{"date-parts":[[2026,5,22]],"date-time":"2026-05-22T03:25:33Z","timestamp":1779420333000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786335.3813164"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,26]]},"references-count":40,"alternative-id":["10.1145\/3786335.3813164","10.1145\/3786335"],"URL":"https:\/\/doi.org\/10.1145\/3786335.3813164","relation":{},"subject":[],"published":{"date-parts":[[2026,5,26]]},"assertion":[{"value":"2026-05-26","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}