{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T08:56:41Z","timestamp":1773392201452,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,11,24]],"date-time":"2024-11-24T00:00:00Z","timestamp":1732406400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS-2024878 and IIS-2145642"],"award-info":[{"award-number":["IIS-2024878 and IIS-2145642"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"Army Research Laboratory","doi-asserted-by":"publisher","award":["W911NF2120076"],"award-info":[{"award-number":["W911NF2120076"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,11,24]]},"DOI":"10.1145\/3687272.3688319","type":"proceedings-article","created":{"date-parts":[[2024,11,20]],"date-time":"2024-11-20T00:24:28Z","timestamp":1732062268000},"page":"54-63","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["GPT-4 as a Moral Reasoner for Robot Command Rejection"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1590-1787","authenticated-orcid":false,"given":"Ruchen","family":"Wen","sequence":"first","affiliation":[{"name":"Interactive Robotics and Language Lab, UMBC, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2413-9368","authenticated-orcid":false,"given":"Francis","family":"Ferraro","sequence":"additional","affiliation":[{"name":"University of Maryland Baltimore County, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1383-8120","authenticated-orcid":false,"given":"Cynthia","family":"Matuszek","sequence":"additional","affiliation":[{"name":"Interactive Robotics and Language Lab, UMBC, United States"}]}],"member":"320","published-online":{"date-parts":[[2024,11,24]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems. 1168\u20131176","author":"Amir Dan","year":"2018","unstructured":"Dan Amir and Ofra Amir. 2018. Highlights: Summarizing agent behavior to people. In Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems. 1168\u20131176."},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems. 1203\u20131207","author":"Amir Ofra","year":"2018","unstructured":"Ofra Amir, Finale Doshi-Velez, and David Sarne. 2018. Agent strategy summarization. In Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems. 1203\u20131207."},{"key":"e_1_3_2_1_3_1","volume-title":"18th International Conference on Autonomous Agents and Multiagent Systems (AAMAS 2019","author":"Anjomshoae Sule","year":"2019","unstructured":"Sule Anjomshoae, Amro Najjar, Davide Calvaresi, and Kary Fr\u00e4mling. 2019. Explainable agents and robots: Results from a systematic literature review. In 18th International Conference on Autonomous Agents and Multiagent Systems (AAMAS 2019), Montreal, Canada, May 13\u201317, 2019. International Foundation for Autonomous Agents and Multiagent Systems, 1078\u20131088."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.2478\/s13230-010-0011-3"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/s43154-020-00026-1"},{"key":"e_1_3_2_1_6_1","volume-title":"Designing sociable robots","author":"Breazeal Cynthia","unstructured":"Cynthia Breazeal. 2004. Designing sociable robots. MIT press."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12369-021-00780-y"},{"key":"e_1_3_2_1_8_1","volume-title":"symposium series.","author":"Briggs Gordon\u00a0Michael","year":"2015","unstructured":"Gordon\u00a0Michael Briggs and Matthias Scheutz. 2015. \u201cSorry, I can\u2019t do that\u201d: Developing Mechanisms to Appropriately Reject Directives in Human-Robot Interactions. In 2015 AAAI fall symposium series."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511813085"},{"key":"e_1_3_2_1_10_1","volume-title":"The 1st international conference on human\u2013agent interaction.","author":"Cormier Derek","year":"2013","unstructured":"Derek Cormier, Gem Newman, Masayuki Nakane, James\u00a0E Young, and Stephane Durocher. 2013. Would you do as a robot commands? An obedience study for human-robot interaction. In The 1st international conference on human\u2013agent interaction."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434073.3444657"},{"key":"e_1_3_2_1_12_1","volume-title":"A survey of socially interactive robots. Robotics and autonomous systems 42, 3-4","author":"Fong Terrence","year":"2003","unstructured":"Terrence Fong, Illah Nourbakhsh, and Kerstin Dautenhahn. 2003. A survey of socially interactive robots. Robotics and autonomous systems 42, 3-4 (2003), 143\u2013166."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12369-021-00811-8"},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. 269\u2013282","author":"Tsung-Yu Hou Yoyo","year":"2024","unstructured":"Yoyo Tsung-Yu Hou, EunJeong Cheon, and Malte\u00a0F Jung. 2024. Power in Human-Robot Interaction. In Proceedings of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. 269\u2013282."},{"key":"e_1_3_2_1_15_1","volume-title":"IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS).","author":"Jackson Ryan\u00a0Blake","year":"2021","unstructured":"Ryan\u00a0Blake Jackson, Sihui Li, Santosh Balajee\u00a0Banisetty, Sriram Siva, Hao Zhang, Neil Dantam, and Tom Williams. 2021. An Integrated Approach to Context-Sensitive Moral Cognition in Robot Cognitive Architectures. In IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314241"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of ICRES","author":"Jackson Ryan\u00a0Blake","year":"2018","unstructured":"Ryan\u00a0Blake Jackson and Tom Williams. 2018. Robot: Asker of questions and changer of norms. Proceedings of ICRES (2018)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/HRI.2019.8673123"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.3389\/frobt.2021.687726"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319502.3374831"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2696454.2696460"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634969"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-8660"},{"key":"e_1_3_2_1_24_1","volume-title":"RO-MAN TSAR Workshop.","author":"Kim Boyoung","year":"2021","unstructured":"Boyoung Kim, Ruchen Wen, Ewart\u00a0J de Visser, Qin Zhu, Tom Williams, and Elizabeth Phillips. 2021. Investigating Robot Moral Advice to Deter Cheating Behavior. In RO-MAN TSAR Workshop."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434074.3446908"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634966"},{"key":"e_1_3_2_1_27_1","volume-title":"Workshops at the Thirtieth AAAI Conference on Artificial Intelligence.","author":"Kuipers Benjamin","year":"2016","unstructured":"Benjamin Kuipers. 2016. Human-like morality and ethics for robots. In Workshops at the Thirtieth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634963"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2157689.2157748"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634999"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Bertram\u00a0F Malle. 2016. Integrating Robot Ethics and Machine Morality: The Study and Design of Moral Competence in Robots. Ethics and Info. Tech. (2016).","DOI":"10.1007\/s10676-015-9367-8"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/2960587.2960597"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.csl.2021.101255"},{"key":"e_1_3_2_1_34_1","volume-title":"Scarecrows in Oz: Large Language Models in HRI. In Companion of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. 1338\u20131340","author":"Matuszek Cynthia","year":"2024","unstructured":"Cynthia Matuszek, Nick Depalma, Ross Mead, Tom Williams, and Ruchen Wen. 2024. Scarecrows in Oz: Large Language Models in HRI. In Companion of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction. 1338\u20131340."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634943"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the HRI Workshop on Scarecrows in Oz: Large Language Models in HRI.","author":"Mott Terran","year":"2024","unstructured":"Terran Mott and Tom Williams. 2024. Hidden Scarecrows: Potential Consequences of Inaccurate Assumptions About LLMs in Robotic Moral Reasoning. In Proceedings of the HRI Workshop on Scarecrows in Oz: Large Language Models in HRI."},{"key":"e_1_3_2_1_37_1","volume-title":"Sayplan: Grounding large language models using 3d scene graphs for scalable task planning. arXiv preprint arXiv:2307.06135","author":"Rana Krishan","year":"2023","unstructured":"Krishan Rana, Jesse Haviland, Sourav Garg, Jad Abou-Chakra, Ian Reid, and Niko Suenderhauf. 2023. Sayplan: Grounding large language models using 3d scene graphs for scalable task planning. arXiv preprint arXiv:2307.06135 (2023)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3029798.3034782"},{"key":"e_1_3_2_1_39_1","volume-title":"Explainability in human\u2013agent systems. Autonomous agents and multi-agent systems 33","author":"Rosenfeld Avi","year":"2019","unstructured":"Avi Rosenfeld and Ariella Richardson. 2019. Explainability in human\u2013agent systems. Autonomous agents and multi-agent systems 33 (2019), 673\u2013705."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3564240"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1080\/01691864.2022.2029720"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415247"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2020.09.198"},{"key":"e_1_3_2_1_44_1","volume-title":"Llm-assist: Enhancing closed-loop planning with language-based reasoning. arXiv preprint arXiv:2401.00125","author":"Sharan SP","year":"2023","unstructured":"SP Sharan, Francesco Pittaluga, Manmohan Chandraker, 2023. Llm-assist: Enhancing closed-loop planning with language-based reasoning. arXiv preprint arXiv:2401.00125 (2023)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3635000"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00280"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319502.3374802"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.2478\/pjbr-2013-0006"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610977.3634990"},{"key":"e_1_3_2_1_50_1","volume-title":"Bayesian inference for psychology. Part II: Example applications with JASP. Psychonomic bulletin & review 25","author":"Wagenmakers Eric-Jan","year":"2018","unstructured":"Eric-Jan Wagenmakers, Jonathon Love, Maarten Marsman, Tahira Jamil, Alexander Ly, Josine Verhagen, Ravi Selker, Quentin\u00a0F Gronau, Damian Dropmann, Bruno Boutin, 2018. Bayesian inference for psychology. Part II: Example applications with JASP. Psychonomic bulletin & review 25 (2018), 58\u201376."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11704-024-40231-1"},{"key":"e_1_3_2_1_52_1","volume-title":"Exploring the reasoning abilities of multimodal large language models (mllms): A comprehensive survey on emerging trends in multimodal reasoning. arXiv preprint arXiv:2401.06805","author":"Wang Yiqi","year":"2024","unstructured":"Yiqi Wang, Wentao Chen, Xiaotian Han, Xudong Lin, Haiteng Zhao, Yongfei Liu, Bohan Zhai, Jianbo Yuan, Quanzeng You, and Hongxia Yang. 2024. Exploring the reasoning abilities of multimodal large language models (mllms): A comprehensive survey on emerging trends in multimodal reasoning. arXiv preprint arXiv:2401.06805 (2024)."},{"key":"e_1_3_2_1_53_1","volume-title":"Jailbroken: How does llm safety training fail?Advances in Neural Information Processing Systems 36","author":"Wei Alexander","year":"2024","unstructured":"Alexander Wei, Nika Haghtalab, and Jacob Steinhardt. 2024. Jailbroken: How does llm safety training fail?Advances in Neural Information Processing Systems 36 (2024)."},{"key":"e_1_3_2_1_54_1","volume-title":"Toward Hybrid Relational-Normative Models of Robot Cognition. In Companion of the 2021 ACM\/IEEE International Conference on Human-Robot Interaction. 568\u2013570","author":"Wen Ruchen","year":"2021","unstructured":"Ruchen Wen. 2021. Toward Hybrid Relational-Normative Models of Robot Cognition. In Companion of the 2021 ACM\/IEEE International Conference on Human-Robot Interaction. 568\u2013570."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/HRI53351.2022.9889594"},{"key":"e_1_3_2_1_56_1","volume-title":"HRI Workshop on the Dark Side of Human-Robot Interaction.","author":"Wen Ruchen","year":"2019","unstructured":"Ruchen Wen, Ryan\u00a0Blake Jackson, Tom Williams, and Qin Zhu. 2019. Towards a role ethics approach to command rejection. In HRI Workshop on the Dark Side of Human-Robot Interaction."},{"key":"e_1_3_2_1_57_1","volume-title":"Comparing Strategies for Robot Communication of Role-Grounded Moral Norms. In Companion of the 2021 ACM\/IEEE International Conference on Human-Robot Interaction. 323\u2013327","author":"Wen Ruchen","year":"2021","unstructured":"Ruchen Wen, Boyoung Kim, Elizabeth Phillips, Qin Zhu, and Tom Williams. 2021. Comparing Strategies for Robot Communication of Role-Grounded Moral Norms. In Companion of the 2021 ACM\/IEEE International Conference on Human-Robot Interaction. 323\u2013327."},{"key":"e_1_3_2_1_58_1","volume-title":"Comparing Norm-Based and Role-Based Strategies for Robot Communication of Role-Grounded Moral Norms. ACM Transactions on Human-Robot Interaction (T-HRI)","author":"Wen Ruchen","year":"2022","unstructured":"Ruchen Wen, Boyoung Kim, Elizabeth Phillips, Qin Zhu, and Tom Williams. 2022. Comparing Norm-Based and Role-Based Strategies for Robot Communication of Role-Grounded Moral Norms. ACM Transactions on Human-Robot Interaction (T-HRI) (2022)."},{"key":"e_1_3_2_1_59_1","volume-title":"Moral Reflections Enhance Robotic Moral Persuasive Capability. In International Conference on Persuasive Technology. Springer, 290\u2013304","author":"Wen Ruchen","year":"2023","unstructured":"Ruchen Wen, Boyoung Kim, Elizabeth Phillips, Qin Zhu, and Tom Williams. 2023. On Further Reflection... Moral Reflections Enhance Robotic Moral Persuasive Capability. In International Conference on Persuasive Technology. Springer, 290\u2013304."},{"key":"e_1_3_2_1_60_1","unstructured":"Tom Williams Ryan\u00a0Blake Jackson and Jane Lockshin. 2018. A Bayesian Analysis of Moral Norm Malleability during Clarification Dialogues.. In CogSci."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3604632"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Tom Williams Cynthia Matuszek Ross Mead and Nick Depalma. 2024. Scarecrows in Oz: The Use of Large Language Models in HRI. 11\u00a0pages.","DOI":"10.1145\/3606261"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3371382.3380740"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434074.3446910"},{"key":"e_1_3_2_1_65_1","volume-title":"Harnessing the power of llms in practice: A survey on chatgpt and beyond. ACM Transactions on Knowledge Discovery from Data 18, 6","author":"Yang Jingfeng","year":"2024","unstructured":"Jingfeng Yang, Hongye Jin, Ruixiang Tang, Xiaotian Han, Qizhang Feng, Haoming Jiang, Shaochen Zhong, Bing Yin, and Xia Hu. 2024. Harnessing the power of llms in practice: A survey on chatgpt and beyond. ACM Transactions on Knowledge Discovery from Data 18, 6 (2024), 1\u201332."},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"crossref","unstructured":"Ziyi Yang Shreyas\u00a0S Raman Ankit Shah and Stefanie Tellex. 2023. Plug in the Safety Chip: Enforcing Temporal Constraints for LLM Agents. (2023).","DOI":"10.1109\/ICRA57147.2024.10611447"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-62056-1_8"},{"key":"e_1_3_2_1_68_1","first-page":"134","article-title":"Role-based Morality, Ethical Pluralism, and Morally Capable Robots","volume":"20","author":"Zhu Qin","year":"2021","unstructured":"Qin Zhu, Tom Williams, and Ruchen Wen. 2021. Role-based Morality, Ethical Pluralism, and Morally Capable Robots. Journal of Contemporary Eastern Asia 20, 1 (2021), 134\u2013150.","journal-title":"Journal of Contemporary Eastern Asia"}],"event":{"name":"HAI '24: International Conference on Human-Agent Interaction","location":"Swansea United Kingdom","acronym":"HAI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 12th International Conference on Human-Agent Interaction"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3687272.3688319","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3687272.3688319","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:39:50Z","timestamp":1755866390000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3687272.3688319"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,24]]},"references-count":68,"alternative-id":["10.1145\/3687272.3688319","10.1145\/3687272"],"URL":"https:\/\/doi.org\/10.1145\/3687272.3688319","relation":{},"subject":[],"published":{"date-parts":[[2024,11,24]]},"assertion":[{"value":"2024-11-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}