{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T17:56:12Z","timestamp":1768413372021,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":73,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,10]],"date-time":"2023-07-10T00:00:00Z","timestamp":1688947200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,10]]},"DOI":"10.1145\/3563657.3596100","type":"proceedings-article","created":{"date-parts":[[2023,7,10]],"date-time":"2023-07-10T01:19:40Z","timestamp":1688951980000},"page":"2281-2303","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["SME-in-the-loop: Interaction Preferences when Supervising Bots in Human-AI Communities"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0686-7911","authenticated-orcid":false,"given":"Zahra","family":"Ashktorab","sequence":"first","affiliation":[{"name":"Thomas J. Watson Center, IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1796-1161","authenticated-orcid":false,"given":"Michael","family":"Desmond","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7199-5493","authenticated-orcid":false,"given":"James M.","family":"Johnson","sequence":"additional","affiliation":[{"name":"AI X-Lab, IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0437-1736","authenticated-orcid":false,"given":"Qian","family":"Pan","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1508-2091","authenticated-orcid":false,"given":"Casey","family":"Dugan","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8152-441X","authenticated-orcid":false,"given":"Michelle","family":"Brachman","sequence":"additional","affiliation":[{"name":"AI Interaction, IBM Research, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-3525-2327","authenticated-orcid":false,"given":"Carolina","family":"Spina","sequence":"additional","affiliation":[{"name":"IBM Argentina, Argentina"}]}],"member":"320","published-online":{"date-parts":[[2023,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1015330.1015430"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025655"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICMLA.2018.00106"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2019.8870185"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITHET50392.2021.9759796"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_2_1_7_1","volume-title":"Director: Generator-classifiers for supervised language modeling. arXiv preprint arXiv:2206.07694","author":"Arora Kushal","year":"2022","unstructured":"Kushal Arora, Kurt Shuster, Sainbayar Sukhbaatar, and Jason Weston. 2022. Director: Generator-classifiers for supervised language modeling. arXiv preprint arXiv:2206.07694 (2022)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300484"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3176349.3176380"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/s43681-021-00110-y"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData47090.2019.9006355"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2501988.2502024"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Michael Bain and Claude Sammut. 1995. A Framework for Behavioural Cloning.. In Machine Intelligence 15. 103\u2013129.","DOI":"10.1093\/oso\/9780198538677.003.0006"},{"key":"e_1_3_2_1_14_1","volume-title":"Asking easy questions: A user-friendly approach to active reward learning. arXiv preprint arXiv:1910.04365","author":"B\u0131y\u0131k Erdem","year":"2019","unstructured":"Erdem B\u0131y\u0131k, Malayandi Palan, Nicholas\u00a0C Landolfi, Dylan\u00a0P Losey, and Dorsa Sadigh. 2019. Asking easy questions: A user-friendly approach to active reward learning. arXiv preprint arXiv:1910.04365 (2019)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2157689.2157693"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300320"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3405755.3406173"},{"key":"e_1_3_2_1_18_1","volume-title":"Investigating Positive and Negative Qualities of Human-in-the-Loop Optimization for Designing Interaction Techniques. In CHI Conference on Human Factors in Computing Systems. 1\u201314","author":"Chan Liwei","year":"2022","unstructured":"Liwei Chan, Yi-Chi Liao, George\u00a0B Mo, John\u00a0J Dudley, Chun-Lien Cheng, Per\u00a0Ola Kristensson, and Antti Oulasvirta. 2022. Investigating Positive and Negative Qualities of Human-in-the-Loop Optimization for Designing Interaction Techniques. In CHI Conference on Human Factors in Computing Systems. 1\u201314."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2019.00075"},{"key":"e_1_3_2_1_20_1","volume-title":"Deep reinforcement learning from human preferences. Advances in neural information processing systems 30","author":"Christiano F","year":"2017","unstructured":"Paul\u00a0F Christiano, Jan Leike, Tom Brown, Miljan Martic, Shane Legg, and Dario Amodei. 2017. Deep reinforcement learning from human preferences. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2006.152"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1177\/1471082X0600700102"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10606-005-9000-1"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2020.103436"},{"key":"e_1_3_2_1_26_1","unstructured":"Jasper Feine Stefan Morana and Alexander Maedche. 2020. Designing Interactive Chatbot Development Systems.. In ICIS."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1111449.1111488"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of The Eleventh International Society for Music Information Retrieval Conference (ISMIR 2010)","author":"Fiebrink Rebecca","year":"2010","unstructured":"Rebecca Fiebrink and Perry\u00a0R Cook. 2010. The Wekinator: a system for real-time, interactive machine learning in music. In Proceedings of The Eleventh International Society for Music Information Retrieval Conference (ISMIR 2010)(Utrecht), Vol.\u00a03."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2767151"},{"key":"e_1_3_2_1_30_1","volume-title":"Learning from dialogue after deployment: Feed yourself, chatbot!arXiv preprint arXiv:1901.05415","author":"Hancock Braden","year":"2019","unstructured":"Braden Hancock, Antoine Bordes, Pierre-Emmanuel Mazare, and Jason Weston. 2019. Learning from dialogue after deployment: Feed yourself, chatbot!arXiv preprint arXiv:1901.05415 (2019)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1093\/jcmc\/zmz022"},{"key":"e_1_3_2_1_32_1","unstructured":"Geert Hofstede. 2009. Geert Hofstede cultural dimensions. (2009)."},{"key":"e_1_3_2_1_33_1","volume-title":"Complexity and Artificial Markets","author":"Hofstede Gert\u00a0Jan","unstructured":"Gert\u00a0Jan Hofstede, Catholijn\u00a0M Jonker, and Tim Verwaart. 2008. Long-term orientation in trade. In Complexity and Artificial Markets. Springer, 107\u2013119."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-73110-8_9"},{"key":"e_1_3_2_1_35_1","volume-title":"Twenty-Sixth AAAI Conference on Artificial Intelligence.","author":"Kapoor Ashish","year":"2012","unstructured":"Ashish Kapoor, Bongshin Lee, Desney Tan, and Eric Horvitz. 2012. Performance and preferences: Interactive refinement of machine learning procedures. In Twenty-Sixth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_36_1","volume-title":"Distilling the knowledge of large-scale generative models into retrieval models for efficient open-domain conversation. arXiv preprint arXiv:2108.12582","author":"Kim Beomsu","year":"2021","unstructured":"Beomsu Kim, Seokjun Seo, Seungju Han, Enkhbayar Erdenee, and Buru Chang. 2021. Distilling the knowledge of large-scale generative models into retrieval models for efficient open-domain conversation. arXiv preprint arXiv:2108.12582 (2021)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376785"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Pallavi Koppol Henny Admoni and Reid\u00a0G Simmons. 2021. Interaction Considerations in Learning from Humans.. In IJCAI. 283\u2013291.","DOI":"10.24963\/ijcai.2021\/40"},{"key":"e_1_3_2_1_39_1","volume-title":"How software developers mitigate collaboration friction with chatbots. arXiv preprint arXiv:1702.07011","author":"Lebeuf Carlene","year":"2017","unstructured":"Carlene Lebeuf, Margaret-Anne Storey, and Alexey Zagalsky. 2017. How software developers mitigate collaboration friction with chatbots. arXiv preprint arXiv:1702.07011 (2017)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300932"},{"key":"e_1_3_2_1_41_1","volume-title":"Human-Robot Interaction (HRI), 2010 5th ACM\/IEEE International Conference on. IEEE, 203\u2013210","author":"Lee Min\u00a0Kyung","year":"2010","unstructured":"Min\u00a0Kyung Lee, Sara Kiesler, Jodi Forlizzi, Siddhartha Srinivasa, and Paul Rybski. 2010. Gracefully mitigating breakdowns in robotic services. In Human-Robot Interaction (HRI), 2010 5th ACM\/IEEE International Conference on. IEEE, 203\u2013210."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2157689.2157691"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173577"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i17.17768"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.3390\/ijerph18042121"},{"key":"e_1_3_2_1_46_1","volume-title":"Conducting behavioral research on Amazon\u2019s Mechanical Turk. Behavior research methods 44, 1","author":"Mason Winter","year":"2012","unstructured":"Winter Mason and Siddharth Suri. 2012. Conducting behavioral research on Amazon\u2019s Mechanical Turk. Behavior research methods 44, 1 (2012), 1\u201323."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/CIC48465.2019.00011"},{"key":"e_1_3_2_1_48_1","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, 2022. Training language models to follow instructions with human feedback. Advances in Neural Information Processing Systems 35 (2022), 27730\u201327744.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_49_1","volume-title":"International Journal \u00abInformation Theories and Applications 22, 3","author":"Pankratova Nataliya","year":"2015","unstructured":"Nataliya Pankratova and Nadezhda Nedashkovskaya. 2015. Methods of evaluation and improvement of consistency of expert pairwise comparison judgements. International Journal \u00abInformation Theories and Applications 22, 3 (2015), 203\u2013223."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.measurement.2020.108048"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11036-020-01530-6"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"Dorsa Sadigh Anca\u00a0D Dragan Shankar Sastry and Sanjit\u00a0A Seshia. 2017. Active preference-based learning of reward functions.","DOI":"10.15607\/RSS.2017.XIII.053"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2819985"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274426"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300680"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376708"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025677"},{"key":"e_1_3_2_1_58_1","unstructured":"Burr Settles. 2009. Active learning literature survey. (2009)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/INCAE.2018.8579372"},{"key":"e_1_3_2_1_60_1","volume-title":"Blenderbot 3: a deployed conversational agent that continually learns to responsibly engage. arXiv preprint arXiv:2208.03188","author":"Shuster Kurt","year":"2022","unstructured":"Kurt Shuster, Jing Xu, Mojtaba Komeili, Da Ju, Eric\u00a0Michael Smith, Stephen Roller, Megan Ung, Moya Chen, Kushal Arora, Joshua Lane, 2022. Blenderbot 3: a deployed conversational agent that continually learns to responsibly engage. arXiv preprint arXiv:2208.03188 (2022)."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12525-019-00359-6"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Emanuel Stoeckli Falk Uebernickel and Walter Brenner. 2018. Exploring affordances of slack integrations and their actualization within enterprises-towards an understanding of how chatbots create value. (2018).","DOI":"10.24251\/HICSS.2018.255"},{"key":"e_1_3_2_1_63_1","volume-title":"Tertiary Online Teaching and Learning","author":"Studente Sylvie","unstructured":"Sylvie Studente and Stephen Ellis. 2020. Enhancing the Online Student Experience Through Creating Learning Communities\u2014The Benefits of Chatbots in Higher Education. In Tertiary Online Teaching and Learning. Springer, 25\u201333."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998345"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2007.09.009"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/2901790.2901817"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v048.i09"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.4236\/jssm.2019.123026"},{"key":"e_1_3_2_1_69_1","unstructured":"Jing Wang Anindya Ghose and Panos Ipeirotis. 2012. Bonus disclosure and choice: what motivates the creation of high-quality paid reviews?. In ICIS. Citeseer."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/CAC.2018.8623505"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274451"},{"key":"e_1_3_2_1_72_1","volume-title":"Dialog-based language learning. Advances in Neural Information Processing Systems 29","author":"Weston E","year":"2016","unstructured":"Jason\u00a0E Weston. 2016. Dialog-based language learning. Advances in Neural Information Processing Systems 29 (2016)."},{"key":"e_1_3_2_1_73_1","volume-title":"IUI Workshops.","author":"Yang Yiwei","year":"2019","unstructured":"Yiwei Yang, Eser Kandogan, Yunyao Li, Prithviraj Sen, and Walter\u00a0S Lasecki. 2019. A study on interaction in human-in-the-loop machine learning for text analytics. In IUI Workshops."},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3382821"}],"event":{"name":"DIS '23: Designing Interactive Systems Conference","location":"Pittsburgh PA USA","acronym":"DIS '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 ACM Designing Interactive Systems Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3563657.3596100","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3563657.3596100","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:08Z","timestamp":1750178288000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3563657.3596100"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,10]]},"references-count":73,"alternative-id":["10.1145\/3563657.3596100","10.1145\/3563657"],"URL":"https:\/\/doi.org\/10.1145\/3563657.3596100","relation":{},"subject":[],"published":{"date-parts":[[2023,7,10]]},"assertion":[{"value":"2023-07-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}