{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,18]],"date-time":"2026-06-18T15:46:24Z","timestamp":1781797584236,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":66,"publisher":"ACM","funder":[{"name":"Research Council of Finland project Subjective Functions","award":["grant 357578"],"award-info":[{"award-number":["grant 357578"]}]},{"name":"European Research Council Advanced Grant","award":["no. 101141916"],"award-info":[{"award-number":["no. 101141916"]}]},{"DOI":"10.13039\/501100023293","name":"Finnish Center for Artificial Intelligence","doi-asserted-by":"publisher","award":["grants 328400, 345604, 341763"],"award-info":[{"award-number":["grants 328400, 345604, 341763"]}],"id":[{"id":"10.13039\/501100023293","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,9,28]]},"DOI":"10.1145\/3746059.3747600","type":"proceedings-article","created":{"date-parts":[[2025,9,27]],"date-time":"2025-09-27T07:49:12Z","timestamp":1758959352000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["DxHF: Providing High-Quality Human Feedback for LLM Alignment with Interactive Decomposition"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8105-0944","authenticated-orcid":false,"given":"Danqing","family":"Shi","sequence":"first","affiliation":[{"name":"Aalto University, Helsinki, Finland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2329-6126","authenticated-orcid":false,"given":"Furui","family":"Cheng","sequence":"additional","affiliation":[{"name":"ETH Z\u00fcrich, Z\u00fcrich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1498-9062","authenticated-orcid":false,"given":"Tino","family":"Weinkauf","sequence":"additional","affiliation":[{"name":"KTH Royal Institute of Technology, Stockholm, Sweden"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2498-7837","authenticated-orcid":false,"given":"Antti","family":"Oulasvirta","sequence":"additional","affiliation":[{"name":"Aalto University, Helsinki, Finland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8526-2613","authenticated-orcid":false,"given":"Mennatallah","family":"El-Assady","sequence":"additional","affiliation":[{"name":"ETH Z\u00fcrich, Z\u00fcrich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,9,27]]},"reference":[{"key":"e_1_3_3_2_2_2","volume-title":"Chat GPT now asking casual users to choose the better of two responses","year":"2024","unstructured":"2024. Chat GPT now asking casual users to choose the better of two responses. https:\/\/www.reddit.com\/r\/DataAnnotationTech\/comments\/1ge0pwf"},{"key":"e_1_3_3_2_3_2","volume-title":"Human preference data about helpfulness and harmlessness","year":"2025","unstructured":"2025. Human preference data about helpfulness and harmlessness. https:\/\/huggingface.co\/datasets\/Anthropic\/hh-rlhf"},{"key":"e_1_3_3_2_4_2","volume-title":"Synthetic Users: Run your user and market research with the most human-like AI participants","year":"2025","unstructured":"2025. Synthetic Users: Run your user and market research with the most human-like AI participants. https:\/\/www.syntheticusers.com\/"},{"key":"e_1_3_3_2_5_2","unstructured":"Josh Achiam Steven Adler Sandhini Agarwal Lama Ahmad Ilge Akkaya Florencia\u00a0Leoni Aleman Diogo Almeida Janko Altenschmidt Sam Altman Shyamal Anadkat et\u00a0al. 2023. Gpt-4 technical report. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.08774 (2023)."},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642016"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"crossref","unstructured":"Lisa\u00a0P Argyle Ethan\u00a0C Busby Nancy Fulda Joshua\u00a0R Gubler Christopher Rytting and David Wingate. 2023. Out of one many: Using language models to simulate human samples. Political Analysis 31 3 (2023) 337\u2013351.","DOI":"10.1017\/pan.2023.2"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"crossref","unstructured":"J\u00a0Scott Armstrong William\u00a0B Denniston\u00a0Jr and Matt\u00a0M Gordon. 1975. The use of the decomposition principle in making judgments. Organizational behavior and human performance 14 2 (1975) 257\u2013263.","DOI":"10.1016\/0030-5073(75)90028-8"},{"key":"e_1_3_3_2_9_2","unstructured":"Amanda Askell Yuntao Bai Anna Chen Dawn Drain Deep Ganguli Tom Henighan Andy Jones Nicholas Joseph Ben Mann Nova DasSarma et\u00a0al. 2021. A general language assistant as a laboratory for alignment. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2112.00861 (2021)."},{"key":"e_1_3_3_2_10_2","first-page":"49","volume-title":"EuroVis (Short Papers)","author":"Asokarajan Bharathi","year":"2017","unstructured":"Bharathi Asokarajan, Ronak Etemadpour, June Abbas, Samuel\u00a0J Huskey, and Chris\u00a0E Weaver. 2017. TexTile: A Pixel-Based Focus+ Context Tool For Analyzing Variants Across Multiple Text Scales.. In EuroVis (Short Papers). 49\u201353."},{"key":"e_1_3_3_2_11_2","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan et\u00a0al. 2022. Training a helpful and harmless assistant with reinforcement learning from human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2204.05862 (2022)."},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1754093"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.3115\/1225403.1225421"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Michael Bostock Vadim Ogievetsky and Jeffrey Heer. 2011. D3 data-driven documents. IEEE transactions on visualization and computer graphics 17 12 (2011) 2301\u20132309.","DOI":"10.1109\/TVCG.2011.185"},{"key":"e_1_3_3_2_15_2","volume-title":"Readings in information visualization: using vision to think","author":"Card Stuart\u00a0K","year":"1999","unstructured":"Stuart\u00a0K Card, Jock Mackinlay, and Ben Shneiderman. 1999. Readings in information visualization: using vision to think. Morgan Kaufmann."},{"key":"e_1_3_3_2_16_2","unstructured":"Tuhin Chakrabarty Philippe Laban and Chien-Sheng Wu. 2024. Can AI writing be salvaged? Mitigating Idiosyncrasies and Improving Human-AI Alignment in the Writing Process through Edits. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.14509 (2024)."},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.474"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641904"},{"key":"e_1_3_3_2_19_2","unstructured":"Paul\u00a0F Christiano Jan Leike Tom Brown Miljan Martic Shane Legg and Dario Amodei. 2017. Deep reinforcement learning from human preferences. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_2_20_2","first-page":"4171","volume-title":"Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, volume 1 (long and short papers)","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. Bert: Pre-training of deep bidirectional transformers for language understanding. In Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, volume 1 (long and short papers). 4171\u20134186."},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"crossref","unstructured":"Danica Dillion Niket Tandon Yuling Gu and Kurt Gray. 2023. Can AI language models replace human participants? Trends in Cognitive Sciences 27 7 (2023) 597\u2013600.","DOI":"10.1016\/j.tics.2023.04.008"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"crossref","unstructured":"Edward\u00a0W Dolch. 1936. A basic sight vocabulary. The Elementary School Journal 36 6 (1936) 456\u2013460.","DOI":"10.1086\/457353"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"crossref","unstructured":"Mennatallah El-Assady Rita Sevastjanova Fabian Sperrle Daniel Keim and Christopher Collins. 2017. Progressive learning of topic modeling parameters: A visual analytics framework. IEEE transactions on visualization and computer graphics 24 1 (2017) 382\u2013391.","DOI":"10.1109\/TVCG.2017.2745080"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/BDVA.2018.8534018"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.580"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3672539.3686760"},{"key":"e_1_3_3_2_27_2","unstructured":"Simret\u00a0Araya Gebreegziabher Yukun Yang Elena\u00a0L Glassman and Toby Jia-Jun Li. 2024. Supporting Co-Adaptive Machine Teaching through Human Concept Learning and Cognitive Theories. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.16561 (2024)."},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642139"},{"key":"e_1_3_3_2_29_2","unstructured":"Jiawei Gu Xuhui Jiang Zhichao Shi Hexiang Tan Xuehao Zhai Chengjin Xu Wei Li Yinghan Shen Shengjie Ma Honghao Liu et\u00a0al. 2024. A survey on llm-as-a-judge. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.15594 (2024)."},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642699"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3650956"},{"key":"e_1_3_3_2_32_2","unstructured":"Dylan Hadfield-Menell Stuart\u00a0J Russell Pieter Abbeel and Anca Dragan. 2016. Cooperative inverse reinforcement learning. Advances in neural information processing systems 29 (2016)."},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3649902.3653521"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2017.8585505"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642314"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"crossref","unstructured":"Minsuk Kahng Ian Tenney Mahima Pushkarna Michael\u00a0Xieyang Liu James Wexler Emily Reif Krystal Kallarackal Minsuk Chang Michael Terry and Lucas Dixon. 2024. LLM Comparator: Interactive Analysis of Side-by-Side Evaluation of Large Language Models. IEEE Transactions on Visualization and Computer Graphics (2024).","DOI":"10.1145\/3613905.3650755"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642216"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_3_2_39_2","unstructured":"Jan Kompatscher Danqing Shi Giovanna Varni Tino Weinkauf and Antti Oulasvirta. 2025. Interactive groupwise comparison for reinforcement learning from human feedback. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2507.04340 (2025)."},{"key":"e_1_3_3_2_40_2","volume-title":"International Conference on Learning Representations","author":"Laidlaw Cassidy","year":"2021","unstructured":"Cassidy Laidlaw and Anca Dragan. 2021. The Boltzmann policy distribution: Accounting for systematic suboptimality in human models. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_41_2","unstructured":"Jan Leike David Krueger Tom Everitt Miljan Martic Vishal Maini and Shane Legg. 2018. Scalable agent alignment via reward modeling: a research direction. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1811.07871 (2018)."},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"crossref","unstructured":"Yang Liu Dan Iter Yichong Xu Shuohang Wang Ruochen Xu and Chenguang Zhu. 2023. G-eval: NLG evaluation using gpt-4 with better human alignment. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.16634 (2023).","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"e_1_3_3_2_43_2","volume-title":"International Joint Conference on Artificial Intelligence 2025","author":"Mertsiotaki Androniki","year":"2025","unstructured":"Androniki Mertsiotaki, Stephanie Hofmann, Sarah Keck, Emily Kratsch, Alexander Daum, and Birgit Popp. 2025. Designing Usable Interfaces for Human Evaluation of LLM-Generated Texts: UX Challenges and Solutions. In International Joint Conference on Artificial Intelligence 2025."},{"key":"e_1_3_3_2_44_2","volume-title":"International Conference on Learning Representations","author":"Metz Yannick","year":"2025","unstructured":"Yannick Metz, Andras Geiszl, Rapha\u00ebl Baur, and Mennatallah El-Assady. 2025. Reward Learning from Multiple Feedback Types. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_45_2","volume-title":"ICML2023 Interactive Learning from Implicit Human Feedback Workshop","author":"Metz Yannick","year":"2023","unstructured":"Yannick Metz, David Lindner, Rapha\u00ebl Baur, Daniel Keim, and Mennatallah El-Assady. 2023. RLHF-Blender: A Configurable Interactive Interface for Learning from Diverse Human Feedback. In ICML2023 Interactive Learning from Implicit Human Feedback Workshop."},{"key":"e_1_3_3_2_46_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"crossref","unstructured":"Volodymyr Mnih Koray Kavukcuoglu David Silver Andrei\u00a0A Rusu Joel Veness Marc\u00a0G Bellemare Alex Graves Martin Riedmiller Andreas\u00a0K Fidjeland Georg Ostrovski et\u00a0al. 2015. Human-level control through deep reinforcement learning. nature 518 7540 (2015) 529\u2013533.","DOI":"10.1038\/nature14236"},{"key":"e_1_3_3_2_48_2","first-page":"145","volume-title":"Proceedings of the human language technology conference of the north american chapter of the association for computational linguistics: Hlt-naacl 2004","author":"Nenkova Ani","year":"2004","unstructured":"Ani Nenkova and Rebecca\u00a0J Passonneau. 2004. Evaluating content selection in summarization: The pyramid method. In Proceedings of the human language technology conference of the north american chapter of the association for computational linguistics: Hlt-naacl 2004. 145\u2013152."},{"key":"e_1_3_3_2_49_2","unstructured":"Long Ouyang Jeffrey Wu Xu Jiang Diogo Almeida Carroll Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray et\u00a0al. 2022. Training language models to follow instructions with human feedback. Advances in neural information processing systems 35 (2022) 27730\u201327744."},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"crossref","unstructured":"Ryan Park Rafael Rafailov Stefano Ermon and Chelsea Finn. 2024. Disentangling length from quality in direct preference optimization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.19159 (2024).","DOI":"10.18653\/v1\/2024.findings-acl.297"},{"key":"e_1_3_3_2_51_2","unstructured":"Rafael Rafailov Archit Sharma Eric Mitchell Christopher\u00a0D Manning Stefano Ermon and Chelsea Finn. 2023. Direct preference optimization: Your language model is secretly a reward model. Advances in Neural Information Processing Systems 36 (2023) 53728\u201353741."},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"crossref","unstructured":"Nils Reimers and Iryna Gurevych. 2019. Sentence-bert: Sentence embeddings using siamese bert-networks. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1908.10084 (2019).","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.1109\/VDS60365.2023.00007"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676450"},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"publisher","DOI":"10.1109\/IROS58592.2024.10801540"},{"key":"e_1_3_3_2_56_2","unstructured":"Nisan Stiennon Long Ouyang Jeffrey Wu Daniel Ziegler Ryan Lowe Chelsea Voss Alec Radford Dario Amodei and Paul\u00a0F Christiano. 2020. Learning to summarize with human feedback. Advances in neural information processing systems 33 (2020) 3008\u20133021."},{"key":"e_1_3_3_2_57_2","doi-asserted-by":"crossref","unstructured":"Hendrik Strobelt Daniela Oelke Bum\u00a0Chul Kwon Tobias Schreck and Hanspeter Pfister. 2015. Guidelines for effective usage of text highlighting techniques. IEEE transactions on visualization and computer graphics 22 1 (2015) 489\u2013498.","DOI":"10.1109\/TVCG.2015.2467759"},{"key":"e_1_3_3_2_58_2","doi-asserted-by":"crossref","unstructured":"Nicole Sultanum Devin Singh Michael Brudno and Fanny Chevalier. 2018. Doccurate: A curation-based approach for clinical text visualization. IEEE transactions on visualization and computer graphics 25 1 (2018) 142\u2013151.","DOI":"10.1109\/TVCG.2018.2864905"},{"key":"e_1_3_3_2_59_2","doi-asserted-by":"publisher","DOI":"10.5555\/551283"},{"key":"e_1_3_3_2_60_2","unstructured":"Rohan Taori Ishaan Gulrajani Tianyi Zhang Yann Dubois Xuechen Li Carlos Guestrin Percy Liang and Tatsunori\u00a0B Hashimoto. 2023. Stanford alpaca: An instruction-following llama model."},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"crossref","unstructured":"Wayne\u00a0A Wickelgren. 1977. Speed-accuracy tradeoff and information processing dynamics. Acta psychologica 41 1 (1977) 67\u201385.","DOI":"10.1016\/0001-6918(77)90012-9"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"crossref","unstructured":"Robert\u00a0F Woolson. 2005. Wilcoxon signed-rank test. Encyclopedia of biostatistics 8 (2005).","DOI":"10.1002\/0470011815.b2a15177"},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.175"},{"key":"e_1_3_3_2_64_2","doi-asserted-by":"crossref","unstructured":"Tariq Yousef and Stefan Janicke. 2020. A survey of text alignment visualization. IEEE transactions on visualization and computer graphics 27 2 (2020) 1149\u20131159.","DOI":"10.1109\/TVCG.2020.3028975"},{"key":"e_1_3_3_2_65_2","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric Xing et\u00a0al. 2023. Judging llm-as-a-judge with mt-bench and chatbot arena. Advances in Neural Information Processing Systems 36 (2023) 46595\u201346623."},{"key":"e_1_3_3_2_66_2","doi-asserted-by":"publisher","DOI":"10.5555\/3104322.3104481"},{"key":"e_1_3_3_2_67_2","doi-asserted-by":"crossref","unstructured":"Richard\u00a0P Zipoli\u00a0Jr. 2017. Unraveling difficult sentences: Strategies to support reading comprehension. Intervention in School and Clinic 52 4 (2017) 218\u2013227.","DOI":"10.1177\/1053451216659465"}],"event":{"name":"UIST '25: The 38th Annual ACM Symposium on User Interface Software and Technology","location":"Busan Republic of Korea","acronym":"UIST '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGGRAPH ACM Special Interest Group on Computer Graphics and Interactive Techniques"]},"container-title":["Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3746059.3747600","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,27]],"date-time":"2025-09-27T22:07:11Z","timestamp":1759010831000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3746059.3747600"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,27]]},"references-count":66,"alternative-id":["10.1145\/3746059.3747600","10.1145\/3746059"],"URL":"https:\/\/doi.org\/10.1145\/3746059.3747600","relation":{},"subject":[],"published":{"date-parts":[[2025,9,27]]},"assertion":[{"value":"2025-09-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}