{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,5]],"date-time":"2025-11-05T10:02:24Z","timestamp":1762336944052,"version":"build-2065373602"},"reference-count":53,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T00:00:00Z","timestamp":1748736000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T00:00:00Z","timestamp":1748736000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,3,12]],"date-time":"2025-03-12T00:00:00Z","timestamp":1741737600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100007601","name":"Horizon 2020","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100007601","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100020134","name":"Programma Operativo Nazionale Ricerca e Competitivit\u00e0","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100020134","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010665","name":"EU Framework Programme for Research and Innovation Marie Sk\u0142odowska-Curie Actions","doi-asserted-by":"publisher","award":["955778"],"award-info":[{"award-number":["955778"]}],"id":[{"id":"10.13039\/100010665","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010661","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100010661","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003407","name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003407","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100021856","name":"Ministero dell'Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","award":["PE0000013"],"award-info":[{"award-number":["PE0000013"]}],"id":[{"id":"10.13039\/501100021856","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Engineering Applications of Artificial Intelligence"],"published-print":{"date-parts":[[2025,6]]},"DOI":"10.1016\/j.engappai.2025.110520","type":"journal-article","created":{"date-parts":[[2025,3,14]],"date-time":"2025-03-14T12:39:17Z","timestamp":1741955957000},"page":"110520","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"C","title":["What is behind the curtain? Increasing transparency in reinforcement learning with human preferences and explanations"],"prefix":"10.1016","volume":"149","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9866-8719","authenticated-orcid":false,"given":"Georgios","family":"Angelopoulos","sequence":"first","affiliation":[]},{"given":"Luigi","family":"Mangiacapra","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1362-8799","authenticated-orcid":false,"given":"Alessandra","family":"Rossi","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8626-5805","authenticated-orcid":false,"given":"Claudia","family":"Di Napoli","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3379-1756","authenticated-orcid":false,"given":"Silvia","family":"Rossi","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.engappai.2025.110520_b1","doi-asserted-by":"crossref","first-page":"1292","DOI":"10.3390\/s21041292","article-title":"Reinforcement learning approaches in social robotics","volume":"21","author":"Akalin","year":"2021","journal-title":"Sensors"},{"key":"10.1016\/j.engappai.2025.110520_b2","doi-asserted-by":"crossref","first-page":"83","DOI":"10.3389\/fnbot.2018.00083","article-title":"System transparency in shared autonomy: A mini review","volume":"12","author":"Alonso","year":"2018","journal-title":"Front. Neurorobot."},{"key":"10.1016\/j.engappai.2025.110520_b3","doi-asserted-by":"crossref","unstructured":"Alshiekh, M., Bloem, R., Ehlers, R., K\u00f6nighofer, B., Niekum, S., Topcu, U., 2018. Safe reinforcement learning via shielding. In: Proceedings of the AAAI Conference on Artificial Intelligence. Vol. 32, pp. 2669\u20132678.","DOI":"10.1609\/aaai.v32i1.11797"},{"key":"10.1016\/j.engappai.2025.110520_b4","first-page":"1","article-title":"Introducing CARESSER: A framework for in situ learning robot social assistance from expert knowledge and demonstrations","author":"Andriella","year":"2022","journal-title":"User Model. User-Adapt. Interact."},{"key":"10.1016\/j.engappai.2025.110520_b5","series-title":"2023 32nd IEEE International Conference on Robot and Human Interactive Communication","first-page":"1922","article-title":"Unveiling the learning curve: enhancing transparency in robot\u2019s learning with inner speech and emotions","author":"Angelopoulos","year":"2023"},{"key":"10.1016\/j.engappai.2025.110520_b6","series-title":"International Conference on Social Robotics","first-page":"394","article-title":"Using theory of mind in explanations for fostering transparency in human-robot interaction","author":"Angelopoulos","year":"2023"},{"key":"10.1016\/j.engappai.2025.110520_b7","series-title":"International Conference on Social Robotics","first-page":"300","article-title":"Transparent interactive reinforcement learning using emotional behaviours","author":"Angelopoulos","year":"2022"},{"key":"10.1016\/j.engappai.2025.110520_b8","series-title":"2023 I-RIM Conference","first-page":"186","article-title":"Robot behaviours for transparent human-robot interaction","author":"Angelopoulos","year":"2023"},{"year":"2018","series-title":"DQN-TAMER: Human-in-the-loop reinforcement learning with intractable feedback","author":"Arakawa","key":"10.1016\/j.engappai.2025.110520_b9"},{"key":"10.1016\/j.engappai.2025.110520_b10","series-title":"International Conference on Advanced Intelligent Systems and Informatics","first-page":"287","article-title":"Autonomous robot navigation and exploration using deep reinforcement learning with Gazebo and ROS","author":"Azar","year":"2023"},{"key":"10.1016\/j.engappai.2025.110520_b11","series-title":"2021 IEEE ICRA","first-page":"3927","article-title":"Towards providing explanations for robot motion planning","author":"Brand\u00e3o","year":"2021"},{"key":"10.1016\/j.engappai.2025.110520_b12","doi-asserted-by":"crossref","unstructured":"Brawer, J., Ghose, D., Candon, K., Qin, M., Roncone, A., V\u00e1zquez, M., Scassellati, B., 2023. Interactive Policy Shaping for Human-Robot Collaboration with Transparent Matrix Overlays. In: Proceedings of the 2023 ACM\/IEEE HRI. pp. 525\u2013533.","DOI":"10.1145\/3568162.3576983"},{"issue":"1535","key":"10.1016\/j.engappai.2025.110520_b13","doi-asserted-by":"crossref","first-page":"3527","DOI":"10.1098\/rstb.2009.0157","article-title":"Role of expressive behaviour for robots that learn from people","volume":"364","author":"Breazeal","year":"2009","journal-title":"Phil. Trans. R. Soc. B"},{"year":"2020","series-title":"Safe and Efficient Inverse Reinforcement Learning","author":"Brown","key":"10.1016\/j.engappai.2025.110520_b14"},{"issue":"3","key":"10.1016\/j.engappai.2025.110520_b15","doi-asserted-by":"crossref","first-page":"7091","DOI":"10.1109\/LRA.2022.3180042","article-title":"Android as a receptionist in a shopping mall using inverse reinforcement learning","volume":"7","author":"Chen","year":"2022","journal-title":"IEEE Robot. Autom. Lett."},{"year":"2020","series-title":"Explainable robotic systems: Interpreting outcome-focused actions in a reinforcement learning scenario","author":"Cruz","key":"10.1016\/j.engappai.2025.110520_b16"},{"issue":"4","key":"10.1016\/j.engappai.2025.110520_b17","doi-asserted-by":"crossref","first-page":"104","DOI":"10.3390\/robotics8040104","article-title":"Accelerating interactive reinforcement learning by human advice for an assembly task by a cobot","volume":"8","author":"De Winter","year":"2019","journal-title":"Robotics"},{"issue":"1","key":"10.1016\/j.engappai.2025.110520_b18","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1177\/0018720816681350","article-title":"From here to autonomy: lessons learned from human\u2013automation research","volume":"59","author":"Endsley","year":"2017","journal-title":"Hum. Factors"},{"issue":"2","key":"10.1016\/j.engappai.2025.110520_b19","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1109\/MRA.2019.2904644","article-title":"Robots and transparency: The multiple dimensions of transparency in the context of robot technologies","volume":"26","author":"Felzmann","year":"2019","journal-title":"IEEE Robot. Autom. Mag."},{"key":"10.1016\/j.engappai.2025.110520_b20","doi-asserted-by":"crossref","DOI":"10.3389\/frobt.2021.772141","article-title":"Lessons learned about designing and conducting studies from hri experts","volume":"8","author":"Fraune","year":"2022","journal-title":"Front. Robot. AI"},{"issue":"3","key":"10.1016\/j.engappai.2025.110520_b21","doi-asserted-by":"crossref","first-page":"3156","DOI":"10.1109\/TNNLS.2023.3243631","article-title":"Reinforcement learning control with knowledge shaping","volume":"35","author":"Gao","year":"2023","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"issue":"1","key":"10.1016\/j.engappai.2025.110520_b22","doi-asserted-by":"crossref","first-page":"32","DOI":"10.1007\/s42761-020-00030-w","article-title":"Reconsidering the duchenne smile: Formalizing and testing hypotheses about eye constriction and positive emotion","volume":"2","author":"Girard","year":"2021","journal-title":"Affect. Sci."},{"key":"10.1016\/j.engappai.2025.110520_b23","article-title":"A survey of communicating robot learning during human-robot interaction","author":"Habibian","year":"2024","journal-title":"Int. J. Robot. Res."},{"key":"10.1016\/j.engappai.2025.110520_b24","article-title":"Inverse reward design","volume":"30","author":"Hadfield-Menell","year":"2017","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.engappai.2025.110520_b25","unstructured":"Hameed, I.A., Tan, Z.-H., Thomsen, N.B., Duan, X., 2016. User acceptance of social robots. In: Proceedings of the Ninth International Conference on Advances in Computer-Human Interactions (ACHI 2016), Venice, Italy. pp. 274\u2013279."},{"issue":"1","key":"10.1016\/j.engappai.2025.110520_b26","first-page":"1","article-title":"A primer for conducting experiments in human\u2013robot interaction","volume":"10","author":"Hoffman","year":"2020","journal-title":"ACM Trans. Hum.- Robot. Interact. (THRI)"},{"key":"10.1016\/j.engappai.2025.110520_b27","series-title":"2021 20th International Conference on Advanced Robotics","first-page":"927","article-title":"Explainable reinforcement learning for human-robot collaboration","author":"Iucci","year":"2021"},{"key":"10.1016\/j.engappai.2025.110520_b28","series-title":"Handbook of Structural Equation Modeling","first-page":"650","article-title":"Bayesian structural equation modeling","author":"Kaplan","year":"2012"},{"key":"10.1016\/j.engappai.2025.110520_b29","series-title":"2018 IEEE First International Conference on Artificial Intelligence and Knowledge Engineering","first-page":"244","article-title":"Stochastic reinforcement learning","author":"Kuang","year":"2018"},{"key":"10.1016\/j.engappai.2025.110520_b30","series-title":"2023 32nd IEEE International Conference on Robot and Human Interactive Communication","first-page":"2120","article-title":"Here\u2019s looking at you, robot: The transparency conundrum in HRI","author":"Lee","year":"2023"},{"year":"2014","series-title":"Legibility of Robot Behavior","author":"Lichtenth\u00e4ler","key":"10.1016\/j.engappai.2025.110520_b31"},{"key":"10.1016\/j.engappai.2025.110520_b32","series-title":"2021 European Conference on Mobile Robots","first-page":"1","article-title":"What is my robot doing? remote supervision to support robots for older adults independent living: a field study","author":"Luperto","year":"2021"},{"key":"10.1016\/j.engappai.2025.110520_b33","doi-asserted-by":"crossref","DOI":"10.1016\/j.chb.2022.107473","article-title":"Explanations and trust: What happens to trust when a robot partner does something unexpected?","volume":"138","author":"Lyons","year":"2023","journal-title":"Comput. Hum. Behav."},{"issue":"1","key":"10.1016\/j.engappai.2025.110520_b34","first-page":"406","article-title":"Human-feedback shield synthesis for perceived safety in deep reinforcement learning","volume":"7","author":"Marta","year":"2021","journal-title":"IEEE RAL"},{"issue":"6","key":"10.1016\/j.engappai.2025.110520_b35","doi-asserted-by":"crossref","first-page":"578","DOI":"10.1109\/THMS.2021.3116119","article-title":"Toward robots\u2019 behavioral transparency of temporal difference reinforcement learning with a human teacher","volume":"51","author":"Matarese","year":"2021","journal-title":"IEEE Trans. Hum.- Mach. Syst."},{"key":"10.1016\/j.engappai.2025.110520_b36","series-title":"Artificial Neural Networks \u2013 ICANN 2006","first-page":"840","article-title":"Reward function and initial values: Better choices for accelerated goal-directed reinforcement learning","author":"Matignon","year":"2006"},{"issue":"4","key":"10.1016\/j.engappai.2025.110520_b37","doi-asserted-by":"crossref","DOI":"10.1016\/j.isci.2021.102371","article-title":"What robots want? Hearing the inner voice of a robot","volume":"24","author":"Pipitone","year":"2021","journal-title":"Iscience"},{"key":"10.1016\/j.engappai.2025.110520_b38","doi-asserted-by":"crossref","unstructured":"Romeo, M., Hern\u00e1ndez Garc\u00eda, D., Jones, R., Cangelosi, A., 2019. Deploying a Deep Learning Agent for HRI with Potential\u201c end-users\u201d at Multiple Sheltered Housing Sites. In: Proceedings of the 7th International Conference on Human-Agent Interaction. pp. 81\u201388.","DOI":"10.1145\/3349537.3351886"},{"issue":"6","key":"10.1016\/j.engappai.2025.110520_b39","doi-asserted-by":"crossref","first-page":"975","DOI":"10.1109\/THMS.2023.3327132","article-title":"Human\u2013robot interaction video sequencing task (HRIVST) for robot\u2019s behavior legibility","volume":"53","author":"Rossi","year":"2023","journal-title":"IEEE Trans. Hum.- Mach. Syst."},{"issue":"3","key":"10.1016\/j.engappai.2025.110520_b40","doi-asserted-by":"crossref","first-page":"25","DOI":"10.3390\/mti7030025","article-title":"A literature survey of how to convey transparency in co-located human\u2013robot interaction","volume":"7","author":"Sch\u00f6tt","year":"2023","journal-title":"Multimodal Technol. Interact."},{"issue":"7","key":"10.1016\/j.engappai.2025.110520_b41","doi-asserted-by":"crossref","DOI":"10.1007\/s12369-020-00667-4","article-title":"Perception and evaluation in human\u2013robot interaction: The human\u2013robot interaction evaluation scale (HRIES)\u2014A multicomponent approach of anthropomorphism","volume":"13","author":"Spatola","year":"2021","journal-title":"Int. J. Soc. Robot."},{"key":"10.1016\/j.engappai.2025.110520_b42","first-page":"34519","article-title":"Optimistic exploration in reinforcement learning using symbolic model estimates","volume":"36","author":"Sreedharan","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"issue":"2","key":"10.1016\/j.engappai.2025.110520_b43","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3365668","article-title":"Transparency about a robot\u2019s lack of human psychological capacities: effects on child-robot perception and relationship formation","volume":"9","author":"Straten","year":"2020","journal-title":"ACM Trans. Hum.- Robot. Interact."},{"key":"10.1016\/j.engappai.2025.110520_b44","series-title":"The 16th IEEE RO-MAN","first-page":"1143","article-title":"A personalized robot companion?-the role of individual differences on spatial preferences in HRI scenarios","author":"Syrdal","year":"2007"},{"issue":"3","key":"10.1016\/j.engappai.2025.110520_b45","doi-asserted-by":"crossref","first-page":"230","DOI":"10.1080\/09540091.2017.1310182","article-title":"Designing and implementing transparency for real time inspection of autonomous robots","volume":"29","author":"Theodorou","year":"2017","journal-title":"Connect. Sci."},{"key":"10.1016\/j.engappai.2025.110520_b46","doi-asserted-by":"crossref","unstructured":"Trinh, T., Chen, H., Brown, D.S., 2024. Autonomous assessment of demonstration sufficiency via bayesian inverse reinforcement learning. In: Proceedings of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. pp. 725\u2013733.","DOI":"10.1145\/3610977.3634984"},{"issue":"2","key":"10.1016\/j.engappai.2025.110520_b47","doi-asserted-by":"crossref","first-page":"49","DOI":"10.3390\/technologies6020049","article-title":"Task engagement as personalization feedback for socially-assistive robots and cognitive training","volume":"6","author":"Tsiakas","year":"2018","journal-title":"Technologies"},{"key":"10.1016\/j.engappai.2025.110520_b48","doi-asserted-by":"crossref","unstructured":"Wachowiak, L., Fenn, A., Kamran, H., Coles, A., Celiktutan, O., Canal, G., 2024. When Do People Want an Explanation from a Robot?. In: Proceedings of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. pp. 752\u2013761.","DOI":"10.1145\/3610977.3634990"},{"key":"10.1016\/j.engappai.2025.110520_b49","article-title":"Transparency in persuasive technology, immersive technology, and online marketing: Facilitating users\u2019 informed decision making and practical implications","author":"Wang","year":"2022","journal-title":"Comput. Hum. Behav."},{"key":"10.1016\/j.engappai.2025.110520_b50","series-title":"2022 17th ACM\/IEEE International Conference on Human-Robot Interaction","first-page":"493","article-title":"Correct me if i\u2019m wrong: Using non-experts to repair reinforcement learning policies","author":"van Waveren","year":"2022"},{"key":"10.1016\/j.engappai.2025.110520_b51","series-title":"2022 IEEE\/IAS Industrial and Commercial Power System Asia","first-page":"671","article-title":"Incorporating constraints in reinforcement learning assisted energy system decision making: A selected review","author":"Wei","year":"2022"},{"key":"10.1016\/j.engappai.2025.110520_b52","doi-asserted-by":"crossref","DOI":"10.3389\/frai.2021.550030","article-title":"Explainable ai and reinforcement learning\u2014a systematic review of current approaches and trends","volume":"4","author":"Wells","year":"2021","journal-title":"Front. Artif. Intell."},{"issue":"3","key":"10.1016\/j.engappai.2025.110520_b53","doi-asserted-by":"crossref","first-page":"254","DOI":"10.1109\/THMS.2019.2925717","article-title":"Agent transparency and reliability in human\u2013robot interaction: The influence on user confidence and perceived reliability","volume":"50","author":"Wright","year":"2019","journal-title":"IEEE Trans. Hum.- Mach. Syst."}],"container-title":["Engineering Applications of Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0952197625005202?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0952197625005202?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,11,5]],"date-time":"2025-11-05T09:59:56Z","timestamp":1762336796000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0952197625005202"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6]]},"references-count":53,"alternative-id":["S0952197625005202"],"URL":"https:\/\/doi.org\/10.1016\/j.engappai.2025.110520","relation":{},"ISSN":["0952-1976"],"issn-type":[{"type":"print","value":"0952-1976"}],"subject":[],"published":{"date-parts":[[2025,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"What is behind the curtain? Increasing transparency in reinforcement learning with human preferences and explanations","name":"articletitle","label":"Article Title"},{"value":"Engineering Applications of Artificial Intelligence","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.engappai.2025.110520","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 The Authors. Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"110520"}}