{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T16:33:40Z","timestamp":1780763620337,"version":"3.54.1"},"reference-count":48,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"11","license":[{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key Research and Development Program of China","award":["2023YFC2705700"],"award-info":[{"award-number":["2023YFC2705700"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U23A20318"],"award-info":[{"award-number":["U23A20318"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U2336211"],"award-info":[{"award-number":["U2336211"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62276195"],"award-info":[{"award-number":["62276195"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62225113"],"award-info":[{"award-number":["62225113"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Science and Technology Major Project of Hubei Province","award":["2024BAB046"],"award-info":[{"award-number":["2024BAB046"]}]},{"DOI":"10.13039\/501100012556","name":"Foundation for Innovative Research Groups of Hubei Province","doi-asserted-by":"publisher","award":["2024AFA017"],"award-info":[{"award-number":["2024AFA017"]}],"id":[{"id":"10.13039\/501100012556","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Research Foundation Singapore and DSO National Laboratories"},{"name":"AI Singapore Programme","award":["AISG2-GC-2023-006"],"award-info":[{"award-number":["AISG2-GC-2023-006"]}]},{"DOI":"10.13039\/501100001381","name":"National Research Foundation Singapore","doi-asserted-by":"publisher","award":["NRF-P2024-001"],"award-info":[{"award-number":["NRF-P2024-001"]}],"id":[{"id":"10.13039\/501100001381","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,11]]},"DOI":"10.1109\/tpami.2025.3590730","type":"journal-article","created":{"date-parts":[[2025,7,18]],"date-time":"2025-07-18T17:45:29Z","timestamp":1752860729000},"page":"9550-9562","source":"Crossref","is-referenced-by-count":2,"title":["Aligning Text-to-Image Diffusion Models With Constrained Reinforcement Learning"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1728-2588","authenticated-orcid":false,"given":"Ziyi","family":"Zhang","sequence":"first","affiliation":[{"name":"School of Computer Science, National Engineering Research Center for Multimedia Software and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sen","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Sydney, Darlington, NSW, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5659-3464","authenticated-orcid":false,"given":"Li","family":"Shen","sequence":"additional","affiliation":[{"name":"School of Cyber Science and Technology, Shenzhen Campus of Sun Yat-sen University, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3180-0484","authenticated-orcid":false,"given":"Yibing","family":"Zhan","sequence":"additional","affiliation":[{"name":"JD Explore Academy, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2296-6370","authenticated-orcid":false,"given":"Yong","family":"Luo","sequence":"additional","affiliation":[{"name":"School of Computer Science, National Engineering Research Center for Multimedia Software and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7532-0496","authenticated-orcid":false,"given":"Han","family":"Hu","sequence":"additional","affiliation":[{"name":"School of Information and Electronics, Beijing Institute of Technology, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0059-8458","authenticated-orcid":false,"given":"Bo","family":"Du","sequence":"additional","affiliation":[{"name":"School of Computer Science, National Engineering Research Center for Multimedia Software and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2751-5114","authenticated-orcid":false,"given":"Yonggang","family":"Wen","sequence":"additional","affiliation":[{"name":"College of Computing and Data Science, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7225-5449","authenticated-orcid":false,"given":"Dacheng","family":"Tao","sequence":"additional","affiliation":[{"name":"College of Computing and Data Science, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","first-page":"2256","article-title":"Deep unsupervised learning using nonequilibrium thermodynamics","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sohl-Dickstein"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"ref3","first-page":"25278","article-title":"LAION-5B: An open large-scale dataset for training next generation image-text models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Schuhmann"},{"key":"ref4","first-page":"15903","article-title":"ImageReward: Learning and evaluating human preferences for text-to-image generation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Xu"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/iccv51070.2023.00200"},{"key":"ref6","article-title":"Aligning text-to-image models using human feedback","author":"Lee","year":"2023"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00200"},{"key":"ref8","first-page":"1","article-title":"RAFT: Reward ranked finetuning for generative foundation model alignment","author":"Dong","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref9","first-page":"79858","article-title":"DPOK: Reinforcement learning for fine-tuning text-to-image diffusion models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Fan"},{"key":"ref10","article-title":"Training diffusion models with reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Black"},{"key":"ref11","first-page":"60396","article-title":"Confronting reward overoptimization for diffusion models: A perspective of inductive and primacy biases","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref12","article-title":"Directly fine-tuning diffusion models on differentiable rewards","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Clark"},{"key":"ref13","article-title":"Aligning text-to-image diffusion models with reward backpropagation","author":"Prabhudesai","year":"2023"},{"key":"ref14","first-page":"10835","article-title":"Scaling laws for reward model overoptimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Gao"},{"key":"ref15","first-page":"23190","article-title":"Understanding plasticity in neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Lyle"},{"key":"ref16","article-title":"Maintaining plasticity via regenerative regularization","author":"Kumar","year":"2023"},{"key":"ref17","article-title":"Revisiting plasticity in visual reinforcement learning: Data, modules and training stages","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ma"},{"key":"ref18","first-page":"16828","article-title":"The primacy bias in deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Nikishin"},{"key":"ref19","first-page":"32145","article-title":"The dormant neuron phenomenon in deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sokar"},{"key":"ref20","first-page":"22","article-title":"Constrained policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Achiam"},{"key":"ref21","article-title":"Reward constrained policy optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Tessler"},{"key":"ref22","first-page":"7555","article-title":"Constrained reinforcement learning has zero duality gap","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Paternain"},{"key":"ref23","article-title":"Projection-based constrained policy optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yang"},{"key":"ref24","first-page":"8378","article-title":"Natural policy gradient primal-dual method for constrained markov decision processes","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ding"},{"key":"ref25","first-page":"13644","article-title":"Constrained variational policy optimization for safe reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Liu"},{"key":"ref26","first-page":"9460","article-title":"Defining and characterizing reward gaming","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Skalse"},{"key":"ref27","article-title":"Inform: Mitigating reward hacking in RLHF via information-theoretic reward modeling","author":"Miao","year":"2024"},{"key":"ref28","first-page":"9133","article-title":"Responsive safety in reinforcement learning by pid lagrangian methods","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Stooke"},{"key":"ref29","article-title":"Safe RLHF: Safe reinforcement learning from human feedback","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dai"},{"key":"ref30","article-title":"Confronting reward model overoptimization with constrained RLHF","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Moskovitz"},{"key":"ref31","article-title":"Implicit under-parameterization inhibits data-efficient deep reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kumar"},{"key":"ref32","article-title":"Understanding and preventing capacity loss in reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lyle"},{"key":"ref33","first-page":"6840","article-title":"Denoising diffusion probabilistic models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ho"},{"key":"ref34","article-title":"Denoising diffusion implicit models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Song"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1201\/9781315140223"},{"key":"ref36","first-page":"36652","article-title":"Pick-a-pic: An open dataset of user preferences for text-to-image generation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Kirstain"},{"key":"ref37","first-page":"267","article-title":"Approximately optimal approximate reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kakade"},{"key":"ref38","article-title":"Proximal policy optimization algorithms","author":"Schulman","year":"2017"},{"key":"ref39","article-title":"LoRA: Low-rank adaptation of large language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hu"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00276"},{"key":"ref41","first-page":"12888","article-title":"Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3399098"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3355414"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3345866"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3432812"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3435448"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3432651"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3344758"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11192800\/11085118.pdf?arnumber=11085118","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,6]],"date-time":"2025-10-06T17:36:46Z","timestamp":1759772206000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11085118\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11]]},"references-count":48,"journal-issue":{"issue":"11"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2025.3590730","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11]]}}}