{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,12]],"date-time":"2026-02-12T17:06:56Z","timestamp":1770916016381,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,3,11]],"date-time":"2024-03-11T00:00:00Z","timestamp":1710115200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"US National Science Foundation","award":["IIS 2132887"],"award-info":[{"award-number":["IIS 2132887"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,3,11]]},"DOI":"10.1145\/3610977.3634925","type":"proceedings-article","created":{"date-parts":[[2024,3,10]],"date-time":"2024-03-10T00:19:00Z","timestamp":1710029940000},"page":"303-312","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Modeling Variation in Human Feedback with User Inputs: An Exploratory Methodology"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-1512-2623","authenticated-orcid":false,"given":"Jindan","family":"Huang","sequence":"first","affiliation":[{"name":"Tufts University, Medford, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8615-3715","authenticated-orcid":false,"given":"Reuben M.","family":"Aronson","sequence":"additional","affiliation":[{"name":"Tufts University, Medford, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4789-9979","authenticated-orcid":false,"given":"Elaine Schaertl","family":"Short","sequence":"additional","affiliation":[{"name":"Tufts University, Medford, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,3,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Dqn-tamer: Human-in-the-loop reinforcement learning with intractable feedback. arXiv preprint arXiv:1810.11748","author":"Arakawa Riku","year":"2018","unstructured":"Riku Arakawa, Sosuke Kobayashi, Yuya Unno, Yuta Tsuboi, and Shin-ichi Maeda. Dqn-tamer: Human-in-the-loop reinforcement learning with intractable feedback. arXiv preprint arXiv:1810.11748, 2018."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-51532-8_10"},{"key":"e_1_3_2_1_3_1","volume-title":"AAAI 2005 workshop on human comprehensible machine learning","author":"Thomaz Andrea Lockerd","year":"2005","unstructured":"Andrea Lockerd Thomaz, Guy Hoffman, and Cynthia Breazeal. Real-time interactive reinforcement learning for robots. In AAAI 2005 workshop on human comprehensible machine learning, 2005."},{"key":"e_1_3_2_1_4_1","volume-title":"Investigating human priors for playing video games. arXiv preprint arXiv:1802.10217","author":"Dubey Rachit","year":"2018","unstructured":"Rachit Dubey, Pulkit Agrawal, Deepak Pathak, Thomas L Griffiths, and Alexei A Efros. Investigating human priors for playing video games. arXiv preprint arXiv:1802.10217, 2018."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11485"},{"key":"e_1_3_2_1_7_1","volume-title":"Deep reinforcement learning from human preferences. arXiv preprint arXiv:1706.03741","author":"Christiano Paul","year":"2017","unstructured":"Paul Christiano, Jan Leike, Tom B Brown, Miljan Martic, Shane Legg, and Dario Amodei. Deep reinforcement learning from human preferences. arXiv preprint arXiv:1706.03741, 2017."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.3006254"},{"key":"e_1_3_2_1_9_1","first-page":"792","volume-title":"Proceedings of the 20th international conference on machine learning (ICML-03)","author":"Wiewiora Eric","year":"2003","unstructured":"Eric Wiewiora, Garrison W Cottrell, and Charles Elkan. Principled methods for advising reinforcement learning agents. In Proceedings of the 20th international conference on machine learning (ICML-03), pages 792--799, 2003."},{"key":"e_1_3_2_1_10_1","volume-title":"Reward learning from human preferences and demonstrations in atari. Advances in neural information processing systems, 31","author":"Ibarz Borja","year":"2018","unstructured":"Borja Ibarz, Jan Leike, Tobias Pohlen, Geoffrey Irving, Shane Legg, and Dario Amodei. Reward learning from human preferences and demonstrations in atari. Advances in neural information processing systems, 31, 2018."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS55552.2023.10342458"},{"key":"e_1_3_2_1_12_1","first-page":"3366","volume-title":"IJCAI","author":"Cederborg Thomas","year":"2015","unstructured":"Thomas Cederborg, Ishaan Grover, Charles L Isbell Jr, and Andrea Lockerd Thomaz. Policy shaping with human teachers. In IJCAI, pages 3366--3372, 2015."},{"key":"e_1_3_2_1_13_1","first-page":"720","volume-title":"Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems","author":"Krening Samantha","year":"2019","unstructured":"Samantha Krening and Karen M Feigh. Newtonian action advice: Integrating human verbal instruction with reinforcement learning. In Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems, pages 720--727, 2019."},{"key":"e_1_3_2_1_14_1","unstructured":"Stephen Casper Xander Davies Claudia Shi Thomas Krendl Gilbert J\u00e9r\u00e9my Scheurer Javier Rando Rachel Freedman Tomasz Korbak David Lindner Pedro Freire et al. Open problems and fundamental limitations of reinforcement learning from human feedback. arXiv preprint arXiv:2307.15217 2023."},{"key":"e_1_3_2_1_15_1","volume-title":"Learning shaping strategies in human-in-the-loop interactive reinforcement learning. arXiv preprint arXiv:1811.04272","author":"Yu Chao","year":"2018","unstructured":"Chao Yu, Tianpei Yang, Wenxuan Zhu, Guangliang Li, et al. Learning shaping strategies in human-in-the-loop interactive reinforcement learning. arXiv preprint arXiv:1811.04272, 2018."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-02675-6_46"},{"key":"e_1_3_2_1_17_1","volume-title":"In Proceedings of IJCAI 2016","author":"Amir Ofra","year":"2016","unstructured":"Ofra Amir, Ece Kamar, Andrey Kolobov, and Barbara Grosz. Interactive teaching strategies for agent training. In In Proceedings of IJCAI 2016, 2016."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1514095.1514115"},{"key":"e_1_3_2_1_19_1","first-page":"1","volume-title":"Interactive reinforcement learning through speech guidance in a domestic scenario. In 2015 international joint conference on neural networks (IJCNN)","author":"Cruz Francisco","year":"2015","unstructured":"Francisco Cruz, Johannes Twiefel, Sven Magg, Cornelius Weber, and Stefan Wermter. Interactive reinforcement learning through speech guidance in a domestic scenario. In 2015 international joint conference on neural networks (IJCNN), pages 1--8. IEEE, 2015."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS47612.2022.9982282"},{"key":"e_1_3_2_1_21_1","volume-title":"Interaction algorithm effect on human experience with reinforcement learning. ACM Transactions on Human-Robot Interaction (THRI), 7(2):1--22","author":"Krening Samantha","year":"2018","unstructured":"Samantha Krening and Karen M Feigh. Interaction algorithm effect on human experience with reinforcement learning. ACM Transactions on Human-Robot Interaction (THRI), 7(2):1--22, 2018."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.3390\/biomimetics6010013"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357236.3395525"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319502.3374824"},{"key":"e_1_3_2_1_25_1","volume-title":"Shared autonomy via deep reinforcement learning. arXiv preprint arXiv:1802.01744","author":"Reddy Siddharth","year":"2018","unstructured":"Siddharth Reddy, Anca D Dragan, and Sergey Levine. Shared autonomy via deep reinforcement learning. arXiv preprint arXiv:1802.01744, 2018."},{"key":"e_1_3_2_1_26_1","volume-title":"Policy shaping: Integrating human feedback with reinforcement learning","author":"Griffith Shane","year":"2013","unstructured":"Shane Griffith, Kaushik Subramanian, Jonathan Scholz, Charles L Isbell, and Andrea L Thomaz. Policy shaping: Integrating human feedback with reinforcement learning. Georgia Institute of Technology, 2013."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1597735.1597738"},{"key":"e_1_3_2_1_28_1","volume-title":"Learning behaviors via human-delivered discrete feedback: modeling implicit feedback strategies to speed up learning. Autonomous agents and multi-agent systems, 30(1):30--59","author":"Loftin Robert","year":"2016","unstructured":"Robert Loftin, Bei Peng, James MacGlashan, Michael L Littman, Matthew E Taylor, Jeff Huang, and David L Roberts. Learning behaviors via human-delivered discrete feedback: modeling implicit feedback strategies to speed up learning. Autonomous agents and multi-agent systems, 30(1):30--59, 2016."},{"key":"e_1_3_2_1_29_1","first-page":"2285","volume-title":"International Conference on Machine Learning","author":"MacGlashan James","year":"2017","unstructured":"James MacGlashan, Mark K Ho, Robert Loftin, Bei Peng, Guan Wang, David L Roberts, Matthew E Taylor, and Michael L Littman. Interactive learning from policy-dependent human feedback. In International Conference on Machine Learning, pages 2285--2294. PMLR, 2017."},{"key":"e_1_3_2_1_30_1","volume-title":"Pebble: Feedback-efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training. arXiv preprint arXiv:2106.05091","author":"Lee Kimin","year":"2021","unstructured":"Kimin Lee, Laura Smith, and Pieter Abbeel. Pebble: Feedback-efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training. arXiv preprint arXiv:2106.05091, 2021."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568162.3576983"},{"key":"e_1_3_2_1_32_1","first-page":"2014","volume-title":"Conference on Robot Learning","author":"Dorsa Sadigh Donald Joseph","year":"2023","unstructured":"Donald Joseph Hejna III and Dorsa Sadigh. Few-shot preference learning for human-in-the-loop rl. In Conference on Robot Learning, pages 2014--2025. PMLR, 2023."},{"key":"e_1_3_2_1_33_1","first-page":"368","volume-title":"Conference on Robot Learning","author":"Hoque Ryan","year":"2023","unstructured":"Ryan Hoque, Lawrence Yunliang Chen, Satvik Sharma, Karthik Dharmarajan, Brijen Thananjeyan, Pieter Abbeel, and Ken Goldberg. Fleet-dagger: Interactive robot fleet learning with scalable human supervision. In Conference on Robot Learning, pages 368--380. PMLR, 2023."},{"key":"e_1_3_2_1_34_1","first-page":"738","volume-title":"Conference on Robot Learning","author":"Zhang Ruohan","year":"2023","unstructured":"Ruohan Zhang, Dhruva Bansal, Yilun Hao, Ayano Hiranaka, Jialu Gao, Chen Wang, Roberto Mart\u00edn-Mart\u00edn, Li Fei-Fei, and Jiajun Wu. A dual representation framework for robot learning with human guidance. In Conference on Robot Learning, pages 738--750. PMLR, 2023."},{"key":"e_1_3_2_1_35_1","volume-title":"RSS 2020","author":"Spencer Jonathan","year":"2020","unstructured":"Jonathan Spencer, Sanjiban Choudhury, Matthew Barnes, Matthew Schmittle, Mung Chiang, Peter Ramadge, and Siddhartha Srinivasa. Learning from interventions: Human-robot interaction as both explicit and implicit feedback. In 16th Robotics: Science and Systems, RSS 2020. MIT Press Journals, 2020."},{"key":"e_1_3_2_1_36_1","volume-title":"6th Annual Conference on Robot Learning","author":"Fitzgerald Tesca","year":"2022","unstructured":"Tesca Fitzgerald, Pallavi Koppol, Patrick Callaghan, Russell Quinlan Jun Hei Wong, Reid Simmons, Oliver Kroemer, and Henny Admoni. Inquire: Interactive querying for user-aware informative reasoning. In 6th Annual Conference on Robot Learning, 2022."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-022-08118-z"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/375735.376334"},{"key":"e_1_3_2_1_39_1","volume-title":"Quantifying the effect of feedback frequency in interactive reinforcement learning for robotic tasks. arXiv preprint arXiv:2207.09845","author":"Harnack Daniel","year":"2022","unstructured":"Daniel Harnack, Julie Pivin-Bachler, and Nicol\u00e1s Navarro-Guerrero. Quantifying the effect of feedback frequency in interactive reinforcement learning for robotic tasks. arXiv preprint arXiv:2207.09845, 2022."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3309772.3309801"},{"key":"e_1_3_2_1_41_1","volume-title":"Cobot: A social reinforcement learning agent. Advances in neural information processing systems, 14","author":"Isbell Charles","year":"2001","unstructured":"Charles Isbell and Christian Shelton. Cobot: A social reinforcement learning agent. Advances in neural information processing systems, 14, 2001."},{"key":"e_1_3_2_1_42_1","volume-title":"Sophie Saskin, and Michael L Littman. Deep reinforcement learning from policy-dependent human feedback. arXiv preprint arXiv:1902.04257","author":"Arumugam Dilip","year":"2019","unstructured":"Dilip Arumugam, Jun Ki Lee, Sophie Saskin, and Michael L Littman. Deep reinforcement learning from policy-dependent human feedback. arXiv preprint arXiv:1902.04257, 2019."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICORR.2011.5975338"},{"key":"e_1_3_2_1_44_1","volume-title":"Cengage Learning","author":"Miltenberger Raymond G","year":"2015","unstructured":"Raymond G Miltenberger. Behavior modification: Principles and procedures. Cengage Learning, 2015."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/817"},{"key":"e_1_3_2_1_46_1","volume-title":"Experiments in socially guided exploration: Lessons learned in building robots that learn with and without human teachers. Connection Science, 20(2--3):91--110","author":"Thomaz Andrea L","year":"2008","unstructured":"Andrea L Thomaz and Cynthia Breazeal. Experiments in socially guided exploration: Lessons learned in building robots that learn with and without human teachers. Connection Science, 20(2--3):91--110, 2008."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/DEVLRN.2008.4640845"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434074.3446361"},{"key":"e_1_3_2_1_49_1","volume-title":"exploit or listen: Combining human feedback and policy model to speed up deep reinforcement learning in 3d worlds. arXiv preprint arXiv:1709.03969","author":"Lin Zhiyu","year":"2017","unstructured":"Zhiyu Lin, Brent Harrison, Aaron Keech, and Mark O Riedl. Explore, exploit or listen: Combining human feedback and policy model to speed up deep reinforcement learning in 3d worlds. arXiv preprint arXiv:1709.03969, 2017."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.tate.2010.07.002"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1957656.1957672"},{"key":"e_1_3_2_1_52_1","article-title":"don't get distracted!","author":"Maggi Gianpaolo","year":"2020","unstructured":"Gianpaolo Maggi, Elena Dell'Aquila, Ilenia Cucciniello, and Silvia Rossi. \"don't get distracted!\": the role of social robots' interaction style on users' cognitive performance, acceptance, and non-compliant behavior. International Journal of Social Robotics, pages 1--13, 2020.","journal-title":"International Journal of Social Robotics, pages 1--13"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/40"}],"event":{"name":"HRI '24: ACM\/IEEE International Conference on Human-Robot Interaction","location":"Boulder CO USA","acronym":"HRI '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2024 ACM\/IEEE International Conference on Human-Robot Interaction"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3610977.3634925","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3610977.3634925","content-type":"text\/html","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3610977.3634925","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3610977.3634925","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,28]],"date-time":"2025-08-28T16:33:42Z","timestamp":1756398822000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3610977.3634925"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,11]]},"references-count":53,"alternative-id":["10.1145\/3610977.3634925","10.1145\/3610977"],"URL":"https:\/\/doi.org\/10.1145\/3610977.3634925","relation":{},"subject":[],"published":{"date-parts":[[2024,3,11]]},"assertion":[{"value":"2024-03-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}