{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T20:22:13Z","timestamp":1792786933153,"version":"4.5.0"},"reference-count":193,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Ningbo Youth Science and Technology Innovation Leading Talent Program","award":["2025QL059"],"award-info":[{"award-number":["2025QL059"]}]},{"name":"Pioneer and Leading Goose\u201d R&D Program of Zhejiang","award":["2025C02037"],"award-info":[{"award-number":["2025C02037"]}]},{"name":"Alibaba-Zhejiang University Joint Research Institute of Frontier Technologies","id":[{"id":"https:\/\/ror.org\/00a2xv884","id-type":"ROR","asserted-by":"crossref"}]},{"name":"Science and Technology Project of State Grid Beijing Electric Power Company"},{"name":"Risk Identification Technology Based on Soft Robots and Artificial Intelligence"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1109\/tpami.2026.3704314","type":"journal-article","created":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T19:43:43Z","timestamp":1781639023000},"page":"13079-13098","source":"Crossref","is-referenced-by-count":0,"title":["A Survey of Direct Preference Optimization: Datasets, Theories, Variants, and Applications"],"prefix":"10.1109","volume":"48","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-2014-0663","authenticated-orcid":false,"given":"Wenyi","family":"Xiao","sequence":"first","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zechuan","family":"Wang","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5859-2588","authenticated-orcid":false,"given":"Leilei","family":"Gan","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5174-5182","authenticated-orcid":false,"given":"Shuai","family":"Zhao","sequence":"additional","affiliation":[{"name":"Nanyang Technological University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zongyue","family":"Li","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruirui","family":"Lei","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wanggui","family":"He","sequence":"additional","affiliation":[{"name":"Alibaba Group","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6062-207X","authenticated-orcid":false,"given":"Luu Anh","family":"Tuan","sequence":"additional","affiliation":[{"name":"Nanyang Technological University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Long","family":"Chan","sequence":"additional","affiliation":[{"name":"Alibaba Group","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hao","family":"Jiang","sequence":"additional","affiliation":[{"name":"Alibaba Group","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6121-0384","authenticated-orcid":false,"given":"Zhou","family":"Zhao","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2139-8807","authenticated-orcid":false,"given":"Fei","family":"Wu","sequence":"additional","affiliation":[{"name":"Zhejiang University","place":["Hangzhou, China"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Introducing ChatGPT","year":"2022"},{"key":"ref2","article-title":"Llama: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref3","article-title":"GPT-4 technical report","year":"2023"},{"key":"ref4","article-title":"Mistral 7b","author":"Jiang","year":"2023"},{"key":"ref5","article-title":"GPT-4V(ision) system card","year":"2023"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.52202\/075280-1516"},{"key":"ref7","article-title":"Gemini: A family of highly capable multimodal models","author":"Team","year":"2023"},{"key":"ref8","article-title":"Qwen-Vl: A versatile vision-language model for understanding, localization, text reading, and beyond","author":"Bai","year":"2023"},{"key":"ref9","article-title":"Scalable agent alignment via reward modeling: A research direction","author":"Leike","year":"2018"},{"key":"ref10","article-title":"A general language assistant as a laboratory for alignment","author":"Askell","year":"2021"},{"key":"ref11","article-title":"Introducing superalignment","year":"2023"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.52202\/068431-2011"},{"key":"ref13","first-page":"50750","article-title":"Safe RLHF: Safe reinforcement learning from human feedback","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dai","year":"2024"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.775"},{"key":"ref15","article-title":"Proximal policy optimization algorithms","author":"Schulman","year":"2017"},{"key":"ref16","article-title":"Open problems and fundamental limitations of reinforcement learning from human feedback","author":"Casper","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref17","first-page":"20941","article-title":"The effects of reward misspecification: Mapping and mitigating misaligned models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Pan","year":"2022"},{"key":"ref18","first-page":"34378","article-title":"Causal confusion and reward misidentification in preference-based reward learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Tien","year":"2023"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.52202\/075280-0482"},{"key":"ref20","article-title":"RAFT: Reward ranked fine tuning for generative foundation model alignment","author":"Dong","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref21","first-page":"54605","article-title":"Statistical rejection sampling improves preference optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liu","year":"2024"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i17.29865"},{"key":"ref23","first-page":"53728","article-title":"Direct preference optimization: Your language model is secretly a reward model","volume-title":"Proc. Adv. Neural Inform. Process. Syst.","volume":"36","author":"Rafailov","year":"2024"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.2307\/2334029"},{"key":"ref25","first-page":"12634","article-title":"Model alignment as prospect theoretic optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ethayarajh","year":"2024"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1154"},{"key":"ref27","first-page":"54983","article-title":"Is DPO superior to PPO for LLM alignment? A comprehensive study","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xu","year":"2025"},{"key":"ref28","first-page":"409","article-title":"Insights into alignment: Evaluating DPO and its variants across multiple tasks","volume-title":"Proc. 63rd Annu. Meeting Assoc. Comput. Linguist.","author":"Saeidi","year":"2024"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.35"},{"key":"ref30","article-title":"Online merging optimizers for boosting rewards and mitigating tax in alignment","author":"Lu","year":"2024"},{"key":"ref31","first-page":"4447","article-title":"A general theoretical paradigm to understand learning from human preferences","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Azar","year":"2024"},{"key":"ref32","first-page":"55204","article-title":"Contrastive preference optimization: Pushing the boundaries of LLM performance in machine translation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xu","year":"2024"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.626"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.52202\/079017-3946"},{"key":"ref35","article-title":"Step-controlled DPO: Leveraging stepwise error for enhanced mathematical reasoning","author":"Lu","year":"2024"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i24.34744"},{"key":"ref37","first-page":"58348","article-title":"Token-level direct preference optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zeng","year":"2024"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3770749"},{"key":"ref39","article-title":"Aligning large language models with human: A survey","author":"Wang","year":"2023"},{"key":"ref40","article-title":"Large language model alignment: A survey","author":"Shen","year":"2023"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.940"},{"key":"ref42","article-title":"Policy optimization in RLHF: The impact of out-of-preference data","author":"Li","year":"2023"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1990"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-70362-1_7"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.447"},{"key":"ref46","article-title":"Towards analyzing and understanding the limitations of DPO: A theoretical perspective","author":"Feng","year":"2024"},{"key":"ref47","article-title":"From $r$r to $Q^{*}$Q*: Your language model is secretly a Q-function","author":"Rafailov","year":"2024"},{"key":"ref48","first-page":"6621","article-title":"Self-play fine-tuning converts weak language models to strong language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chen","year":"2024"},{"key":"ref49","article-title":"Understanding the performance gap between online and offline alignment algorithms","author":"Tang","year":"2024"},{"key":"ref50","first-page":"2109","article-title":"Offline RLHF methods need more accurate supervision signals","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Wang","year":"2024"},{"key":"ref51","article-title":"Some things are more cringe than others: Iterative preference optimization with the pairwise cringe loss","author":"Xu","year":"2024"},{"key":"ref52","article-title":"Direct language model alignment from online AI feedback","author":"Guo","year":"2024"},{"key":"ref53","first-page":"57905","article-title":"Self-rewarding language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yuan","year":"2024"},{"key":"ref54","article-title":"Optune: Efficient online preference tuning","author":"Chen","year":"2024"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.52202\/075280-1308"},{"key":"ref56","article-title":"A long way to go: Investigating length correlations in RLHF","volume-title":"Proc. Conf. Lang. Model.","author":"Singhal","year":"2024"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642596"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.52202\/075280-3268"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.297"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.emnlp-main.1233"},{"key":"ref61","article-title":"Length desensitization in directed preference optimization","author":"Liu","year":"2024"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i26.34963"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.85"},{"key":"ref64","article-title":"Learning to reason with LLMs","year":"2024"},{"key":"ref65","article-title":"Chameleon: Mixed-modal early-fusion foundation models","year":"2024"},{"key":"ref66","first-page":"71865","article-title":"Value-incentivized preference optimization: A unified approach to online and offline RLHF","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Cen","year":"2025"},{"key":"ref67","first-page":"24897","article-title":"Advancing LLM reasoning generalists with preference trees","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yuan","year":"2025"},{"key":"ref68","first-page":"366","article-title":"SDPO: Don\u2019t use your data all at once","volume-title":"Proc. Int. Conf. Comput. Linguist.","author":"Kim","year":"2025"},{"key":"ref69","article-title":"Step-DPO: Step-wise preference optimization for long-chain reasoning of LLMs","author":"Lai","year":"2024"},{"key":"ref70","first-page":"25901","article-title":"Building math agents with multi-turn iterative preference learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Xiong","year":"2025"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.52202\/079017-3702"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.52202\/079017-4128"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.52202\/085713-5605"},{"key":"ref74","first-page":"59414","article-title":"Learn your reference model for real good alignment","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Gorbatovski","year":"2025"},{"key":"ref75","first-page":"70523","article-title":"Learning dynamics of LLM finetuning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ren","year":"2025"},{"key":"ref76","article-title":"Online DPO: Online direct preference optimization with fast-slow chasing","author":"Qi","year":"2024"},{"key":"ref77","article-title":"Hybrid preference optimization for alignment: Provably faster convergence rates by combining offline preferences with online exploration","author":"Bose","year":"2025"},{"key":"ref78","first-page":"80338","article-title":"Sail into the headwind: Alignment via robust rewards and dynamic labels against reward hacking","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Rashidinejad","year":"2025"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.1150"},{"key":"ref80","article-title":"PAFT: A parallel training paradigm for effective LLM fine-tuning","author":"Pentyala","year":"2024"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.175"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.630"},{"key":"ref83","first-page":"68364","article-title":"Restoring calibration for aligned large language models: A calibration-aware fine-tuning approach","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xiao","year":"2025"},{"key":"ref84","first-page":"3008","article-title":"Learning to summarize with human feedback","volume-title":"Proc. Adv. Neural Inform. Process. Syst.","volume":"33","author":"Stiennon","year":"2020"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.63317\/2h4pkr685mkq"},{"key":"ref86","article-title":"WebGPT: Browser-assisted question-answering with human feedback","author":"Nakano","year":"2022"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.185"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2064"},{"key":"ref89","first-page":"5988","article-title":"Understanding dataset difficulty with $\\mathcal {V}$V-usable information","volume-title":"Proc. Int. Conf. Mach. Learn.","volume":"162","author":"Ethayarajh","year":"2022"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0047"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01310"},{"key":"ref92","article-title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","author":"Bai","year":"2022"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2020"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.52202\/085713-1448"},{"key":"ref95","first-page":"76625","article-title":"MM-RLHF: The next step forward in multimodal LLM alignment","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang","year":"2025"},{"key":"ref96","article-title":"Towards harmless multimodal assistants with blind","author":"Li","year":"2025"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.01861"},{"key":"ref98","first-page":"9722","article-title":"Ultrafeedback: Boosting language models with scaled AI feedback","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cui","year":"2024"},{"key":"ref99","article-title":"Distilabel-capybara-dpo-7k-binarized","year":"2024"},{"key":"ref100","article-title":"ChatML DPO pairs","author":"Labonne","year":"2023"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.96"},{"key":"ref102","first-page":"76346","article-title":"Magpie: Alignment data synthesis from scratch by prompting aligned LLMs with nothing","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Xu","year":"2025"},{"key":"ref103","article-title":"Zsql-postgres-dpo","year":"2024"},{"key":"ref104","article-title":"Starling-7B: Improving helpfulness and harmlessness with RLAIF","volume-title":"Proc. 1st Conf. Lang. Model.","author":"Zhu","year":"2024"},{"key":"ref105","article-title":"SlimOrca: An open dataset of GPT-4 augmented FLAN reasoning traces, with verification","author":"Lian","year":"2023"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.01850"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.358"},{"key":"ref108","article-title":"Truthy DPO","author":"Durbin","year":"2024"},{"key":"ref109","article-title":"Orca: Progressive learning from complex explanation traces of GPT-4","author":"Mukherjee","year":"2023"},{"key":"ref110","article-title":"Dataset-tldr-preference-dpo","author":"van Strien","year":"2024"},{"key":"ref111","article-title":"Rewardeval: Advancing reward model evaluation","author":"Malik","year":"2026"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.02296"},{"key":"ref113","first-page":"8184","article-title":"CodeSteer: Symbolic-augmented language models via code\/text guidance","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chen","year":"2025"},{"key":"ref114","article-title":"Mask-DPO: Generalizable fine-grained factuality alignment of LLMs","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Gu","year":"2025"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.52202\/085713-5315"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51701.2025.00110"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.643"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.539"},{"key":"ref119","first-page":"39286","article-title":"Self-Play with execution feedback: Improving instruction-following capabilities of large language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dong","year":"2025"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1145\/3746252.3761322"},{"key":"ref121","article-title":"Conifer: Improving complex constrained instruction-following ability of large language models","author":"Sun","year":"2024"},{"key":"ref122","first-page":"11029","article-title":"FIPO: Free-form instruction-oriented prompt optimization with preference dataset and modular fine-tuning schema","volume-title":"Proc. Int. Conf. Comput. Linguist.","author":"Lu","year":"2025"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-naacl.108"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.52202\/079017-3671"},{"key":"ref125","article-title":"Halu-j: Critique-based hallucination judge","author":"Wang","year":"2024"},{"key":"ref126","first-page":"20437","article-title":"Fine-grained hallucination detection and mitigation in long-form question answering","volume-title":"Proc. Annu. Meet. Assoc. Comput. Linguist.","author":"Sachdeva","year":"2025"},{"key":"ref127","first-page":"44851","article-title":"Fine-tuning language models for factuality","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Tian","year":"2024"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.5"},{"key":"ref129","article-title":"Stable code technical report","author":"Pinnaparaju","year":"2024"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/tpds.2026.3675550"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-97-5669-8_11"},{"key":"ref132","article-title":"Aligning modalities in vision large language models via preference fine-tuning","author":"Zhou","year":"2024"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-long.30"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72684-2_9"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00786"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00854"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1145\/3664647.3681688"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49660.2025.10888737"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1594"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.595"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW63382.2024.00195"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.138"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.607"},{"key":"ref144","article-title":"Solving the granularity mismatch: Hierarchical preference learning for long-horizon LLM agents","author":"Gao","year":"2026"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.542"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.emnlp-main.484"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.52202\/085713-1235"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.52202\/085713-2029"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0863"},{"key":"ref150","article-title":"Finetuning large language model for personalized ranking","author":"Bai","year":"2024"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.3390\/pharmaceutics16081014"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW63382.2024.00517"},{"key":"ref153","doi-asserted-by":"crossref","DOI":"10.1101\/2024.05.20.595026","article-title":"Aligning protein generative models with experimental fitness via direct preference optimization","author":"Widatalla","year":"2024"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.2307\/2334029"},{"key":"ref155","article-title":"Token-importance guided direct preference optimization","author":"Yang","year":"2026"},{"key":"ref156","article-title":"ActiveDPO: Active direct preference optimization for sample-efficient alignment","author":"Lin","year":"2026"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.52202\/085713-2362"},{"key":"ref158","first-page":"37906","article-title":"Critical tokens matter: Token-level contrastive estimation enhances llm\u2019s reasoning capability","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Lin","year":"2025"},{"key":"ref159","first-page":"43632","article-title":"Exploratory preference optimization: Harnessing implicit Q*-approximation for sample-efficient RLHF","volume-title":"in Proc. Int. Conf. Learn. Representations","author":"Xie","year":"2025"},{"key":"ref160","first-page":"77403","article-title":"3D-properties: Identifying challenges in DPO and charting a path forward","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yan","year":"2025"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.52202\/085713-2362"},{"key":"ref162","doi-asserted-by":"publisher","DOI":"10.1007\/BF00122574"},{"key":"ref163","article-title":"Smaug: Fixing failure modes of preference optimisation with DPO-positive","author":"Pal","year":"2024"},{"key":"ref164","article-title":"Self-play preference optimization for language model alignment","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Wu","year":"2025"},{"key":"ref165","article-title":"Direct nash optimization: teaching language models to self-improve with general preferences","author":"Rosset","year":"2024"},{"key":"ref166","first-page":"36743","article-title":"Nash learning from human feedback","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Munos","year":"2024"},{"key":"ref167","first-page":"67793","article-title":"AlphaDPO: Adaptive reward margin for direct preference optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wu","year":"2025"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.493"},{"key":"ref169","first-page":"39255","article-title":"The crucial role of samplers in online direct preference optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shi","year":"2025"},{"key":"ref170","article-title":"Trust region preference approximation: A simple and stable reinforcement learning algorithm for LLM reasoning","author":"Su","year":"2025"},{"key":"ref171","article-title":"Evolving LLMs\u2019 self-refinement capability via synergistic training-inference optimization","author":"Zeng","year":"2025"},{"key":"ref172","article-title":"Sight beyond text: Multi-modal training enhances llms in truthfulness and ethics","author":"Tu","year":"2024","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref173","first-page":"1","article-title":"The claude 3 model family: Opus, sonnet, haiku","volume":"1","author":"Anthropic","year":"2024","journal-title":"Claude-3 Model Card"},{"key":"ref174","article-title":"Continual learning for large language models: A survey","author":"Wu","year":"2024"},{"key":"ref175","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v40i45.41215"},{"key":"ref176","article-title":"Gutenberg DPO","author":"Durbin","year":"2024"},{"key":"ref177","article-title":"IMDB-DPO","author":"Sosnin","year":"2024"},{"key":"ref178","article-title":"Hugging Face H4 Stack Exchange preference dataset","author":"Lambert","year":"2023"},{"key":"ref179","article-title":"The Llama 3 herd of models","year":"2024"},{"key":"ref180","article-title":"Qwen2 technical report","author":"Yang","year":"2024"},{"key":"ref181","article-title":"Bridging SFT and DPO for diffusion model alignment with self-sampling preference optimization","author":"Zhang","year":"2024"},{"key":"ref182","first-page":"26874","article-title":"RLAIF vs. RLHF: Scaling reinforcement learning from human feedback with AI feedback","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","volume":"235","author":"Lee","year":"2024"},{"key":"ref183","first-page":"4971","article-title":"Weak-to-strong generalization: Eliciting strong capabilities with weak supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Burns","year":"2024"},{"key":"ref184","article-title":"Weak-to-strong backdoor attacks for LLMs with contrastive knowledge distillation","author":"Zhao","year":"2024"},{"key":"ref185","article-title":"LLM critics help catch LLM bugs","author":"McAleese","year":"2024"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2574"},{"key":"ref187","first-page":"54715","article-title":"Iterative preference learning from human feedback: Bridging theory and practice for RLHF under kl-constraint","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xiong","year":"2024"},{"key":"ref188","first-page":"18129","article-title":"Tool-augmented reward modeling","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Li","year":"2024"},{"key":"ref189","first-page":"11354","article-title":"UNIFIED-IO: A unified model for vision, language, and multi-modal tasks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lu","year":"2023"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i16.33882"},{"key":"ref191","first-page":"28240","article-title":"Show-o: One single transformer to unify multimodal understanding and generation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Xie","year":"2025"},{"key":"ref192","article-title":"Video diffusion alignment via reward gradients","author":"Prabhudesai","year":"2024"},{"key":"ref193","article-title":"A survey on long video generation: Challenges, methods, and prospects","author":"Li","year":"2024"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11674301\/11568678.pdf?arnumber=11568678","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,28]],"date-time":"2026-09-28T19:11:29Z","timestamp":1790622689000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11568678\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":193,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2026.3704314","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,10]]}}}