{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,15]],"date-time":"2026-08-15T17:35:43Z","timestamp":1786815343060,"version":"3.56.0"},"reference-count":104,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Analytical Center for the Government of Russian Federation","award":["70-2021-00143 01.11.2021"],"award-info":[{"award-number":["70-2021-00143 01.11.2021"]}]},{"name":"Analytical Center for the Government of Russian Federation","award":["IGK 000000D730324P540002"],"award-info":[{"award-number":["IGK 000000D730324P540002"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2024]]},"DOI":"10.1109\/access.2024.3504735","type":"journal-article","created":{"date-parts":[[2024,11,22]],"date-time":"2024-11-22T19:05:57Z","timestamp":1732302357000},"page":"175473-175500","source":"Crossref","is-referenced-by-count":86,"title":["Comprehensive Overview of Reward Engineering and Shaping in Advancing Reinforcement Learning Applications"],"prefix":"10.1109","volume":"12","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-9965-8209","authenticated-orcid":false,"given":"Sinan","family":"Ibrahim","sequence":"first","affiliation":[{"name":"Skolkovo Institute of Science and Technology, Moscow, Russia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4047-3878","authenticated-orcid":false,"given":"Mostafa","family":"Mostafa","sequence":"additional","affiliation":[{"name":"Skolkovo Institute of Science and Technology, Moscow, Russia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-3067-9410","authenticated-orcid":false,"given":"Ali","family":"Jnadi","sequence":"additional","affiliation":[{"name":"Institute of Robotics and Computer Vision, Innopolis University, Innopolis, Russia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hadi","family":"Salloum","sequence":"additional","affiliation":[{"name":"Research Center for Artificial Intelligence, Innopolis University, Innopolis, Russia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6184-3293","authenticated-orcid":false,"given":"Pavel","family":"Osinenko","sequence":"additional","affiliation":[{"name":"Skolkovo Institute of Science and Technology, Moscow, Russia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.2307\/1170677"},{"key":"ref2","first-page":"20118","article-title":"Explicable reward design for reinforcement learning agents","volume":"34","author":"Devidze","year":"2021","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-41188-6_3"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1613\/jair.301"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/tnn.1998.712192"},{"key":"ref6","first-page":"1","article-title":"Reinforcement learning and the reward engineering principle","volume-title":"Proc. AAAI Spring Symp. Ser.","author":"Dewey"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1016\/S0149-7634(89)80028-4"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1162\/089976600300015961"},{"key":"ref9","volume-title":"Theory and Application of Reward Shaping in Reinforcement Learning","author":"Laud","year":"2004"},{"key":"ref10","first-page":"15931","article-title":"Learning to utilize shaping rewards: A new approach of reward shaping","volume-title":"Proc. Adv. Neural Inf. Process. Syst. Annu. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Hu"},{"key":"ref11","first-page":"278","article-title":"Policy invariance under reward transformations: Theory and application to reward shaping","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ng"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3605764.3623916"},{"key":"ref13","article-title":"Uncertainty-penalized reinforcement learning from human feedback with diverse reward LoRA ensembles","author":"Zhai","year":"2023","journal-title":"arXiv:2401.00243"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2019.XV.073"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-16952-6_49"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.ifacol.2022.07.619"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2021.02.003"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103535"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1007\/s10458-022-09575-5"},{"key":"ref20","first-page":"225","article-title":"Theoretical considerations of potential-based reward shaping for multi-agent systems","volume-title":"Proc. 10th Int. Conf. Auto. Agents Multi-Agent Syst.","author":"Devlin"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i6.20550"},{"key":"ref22","first-page":"208","article-title":"Value-based policy teaching with active indirect elicitation","volume-title":"Proc. AAAI","volume":"8","author":"Zhang"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/1566374.1566417"},{"key":"ref24","first-page":"19","article-title":"Intrinsically motivated learning of hierarchical collections of skills","volume-title":"Proc. 3rd Int. Conf. Develop. Learn.","volume":"112","author":"Barto"},{"key":"ref25","first-page":"2601","article-title":"Where do rewards come from","volume-title":"Proc. Annu. Conf. Cognit. Sci. Soc.","author":"Singh"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TAMD.2010.2051031"},{"key":"ref27","article-title":"Natural policy gradients in reinforcement learning explained","author":"van Heeswijk","year":"2022","journal-title":"arXiv:2209.01820"},{"key":"ref28","first-page":"1","article-title":"Reward design via online gradient ascent","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Sorg"},{"key":"ref29","first-page":"1","article-title":"On learning intrinsic rewards for policy gradient methods","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Zheng"},{"key":"ref30","article-title":"Leveraging demonstrations for deep reinforcement learning on robotics problems with sparse rewards","author":"Vecerik","year":"2017","journal-title":"arXiv:1707.08817"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/OJITS.2022.3172981"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6086"},{"key":"ref33","article-title":"Robust reward design for Markov decision processes","author":"Wu","year":"2024","journal-title":"arXiv:2406.05086"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/TCST.2024.3393210"},{"key":"ref35","article-title":"LESSON: Learning to integrate exploration strategies for reinforcement learning via an option framework","author":"Kim","year":"2023","journal-title":"arXiv:2310.03342"},{"key":"ref36","first-page":"1","article-title":"# exploration: A study of count-based exploration for deep reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Tang"},{"key":"ref37","first-page":"1","article-title":"Vime: Variational information maximizing exploration","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Houthooft"},{"key":"ref38","first-page":"1329","article-title":"Benchmarking deep reinforcement learning for continuous control","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Duan"},{"key":"ref39","first-page":"5829","article-title":"Exploration-guided reward shaping for reinforcement learning under sparse rewards","volume-title":"Proc. 36th Conf. Adv. Neural Inf. Process. Syst. (NeurIPS)","volume":"35","author":"Devidze"},{"key":"ref40","article-title":"Reward uncertainty for exploration in preference-based reinforcement learning","author":"Liang","year":"2022","journal-title":"arXiv:2205.12401"},{"key":"ref41","first-page":"13957","article-title":"Reparameterized policy learning for multimodal trajectory optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Huang"},{"key":"ref42","first-page":"1","article-title":"Towards generalization and simplicity in continuous control","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Rajeswaran"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103500"},{"key":"ref44","first-page":"1","article-title":"Algorithms for inverse reinforcement learning","volume-title":"Proc. ICML","volume":"1","author":"Ng"},{"key":"ref45","first-page":"1930","article-title":"Model-based inverse reinforcement learning from visual demonstrations","volume-title":"Proc. Conf. Robot Learn.","author":"Das"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2022.110242"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/s11063-017-9702-7"},{"key":"ref48","article-title":"Maximum entropy deep inverse reinforcement learning","author":"Wulfmeier","year":"2015","journal-title":"arXiv:1507.04888"},{"key":"ref49","first-page":"1","article-title":"Inverse reward design","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Hadfield-Menell"},{"key":"ref50","first-page":"440","article-title":"The influence of reward on the speed of reinforcement learning: An analysis of shaping","volume-title":"Proc. 20th Int. Conf. Mach. Learn. (ICML-03)","author":"Laud"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/CCWC57344.2023.10099211"},{"key":"ref52","first-page":"1","article-title":"Potential based reward shaping for hierarchical reinforcement learning","volume-title":"Proc. 24rth Int. Joint Conf. Artif. Intell.","author":"Gao"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v29i1.9628"},{"key":"ref54","volume-title":"Reward Shaping in Episodic Reinforcement Learning","author":"Grzes","year":"2017"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-33170-1_8"},{"key":"ref56","first-page":"165","article-title":"Potential-based difference rewards for multiagent reinforcement learning","volume-title":"Proc. Int. Conf. Auto. Agents Multi-Agent Syst.","author":"Devlin"},{"key":"ref57","first-page":"433","article-title":"Dynamic potential-based reward shaping","volume-title":"Proc. 11th Int. Conf. Auto. Agents Multiagent Syst.","author":"Devlin"},{"key":"ref58","first-page":"263","article-title":"Minimax regret bounds for reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Azar"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/1102351.1102459"},{"key":"ref60","first-page":"15281","article-title":"Unpacking reward shaping: Understanding the benefits of reward engineering on sample complexity","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Gupta"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1209\/epl\/i2000-00208-x"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1017\/S0269888918000292"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/BRACIS.2014.53"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/IS.2008.4670492"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1080\/09540091.2014.885283"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11741"},{"key":"ref67","article-title":"PEBBLE: Feedback-efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training","author":"Lee","year":"2021","journal-title":"arXiv:2106.05091"},{"key":"ref68","article-title":"SURF: Semi-supervised reward learning with data augmentation for feedback-efficient preference-based reinforcement learning","author":"Park","year":"2022","journal-title":"arXiv:2203.10050"},{"key":"ref69","first-page":"1","article-title":"Deep reinforcement learning from human preferences","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Christiano"},{"key":"ref70","first-page":"1","article-title":"Reward learning from human preferences and demonstrations in atari","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ibarz"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-23780-5_11"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CAC.2017.8243575"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.15348"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10945"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-022-10246-w"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2022.05.014"},{"key":"ref77","first-page":"1","article-title":"Text2reward: Reward shaping with language models for reinforcement learning","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"Xie"},{"key":"ref78","first-page":"22270","article-title":"Meta-reward-net: Implicitly differentiable reward learning for preference-based reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Liu"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1103\/PRXEnergy.1.033005"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1103\/PRXEnergy.2.043007"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA57147.2024.10610391"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/331"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i9.16975"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/CoG51982.2022.9893651"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/DCNA59899.2023.10290178"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/CoDIT62066.2024.10708603"},{"key":"ref87","article-title":"Combining automated optimisation of hyperparameters and reward shape","author":"Dierkes","year":"2024","journal-title":"arXiv:2406.18293"},{"key":"ref88","article-title":"DEHB: Evolutionary hyperband for scalable, robust and efficient hyperparameter optimization","author":"Awad","year":"2021","journal-title":"arXiv:2105.09821"},{"key":"ref89","article-title":"Automated reinforcement learning: An overview","author":"Refaei Afshar","year":"2022","journal-title":"arXiv:2201.05000"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1016\/j.aei.2021.101360"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.3390\/app10165574"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-022-00573-6"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2022.3161699"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.3390\/biomimetics9010051"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2024.109147"},{"key":"ref96","first-page":"1","article-title":"Quantum congestion-focused traffic optimization (Q-CFTO) Enhancing traffic congestion solutions with quantum annealing","volume":"2","author":"Salloum","year":"2024","journal-title":"Authorea Preprints"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.3389\/fict.2017.00029"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/SSCI47803.2020.9308468"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2021.3054625"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3124466"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA48891.2023.10161062"},{"key":"ref102","article-title":"Sim-to-real transfer of deep reinforcement learning agents for online coverage path planning","author":"Jonnarth","year":"2024","journal-title":"arXiv:2406.04920"},{"key":"ref103","article-title":"Concrete problems in AI safety","author":"Amodei","year":"2016","journal-title":"arXiv:1606.06565"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.3390\/s23073762"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/10380310\/10763475.pdf?arnumber=10763475","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,29]],"date-time":"2024-11-29T07:39:26Z","timestamp":1732865966000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10763475\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":104,"URL":"https:\/\/doi.org\/10.1109\/access.2024.3504735","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}