{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,19]],"date-time":"2026-09-19T08:52:33Z","timestamp":1789807953323,"version":"4.0.1"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,3,23]]},"DOI":"10.1145\/3742413.3789065","type":"proceedings-article","created":{"date-parts":[[2026,3,3]],"date-time":"2026-03-03T11:32:24Z","timestamp":1772537544000},"page":"599-616","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Evaluating Generative AI in the Lab: Methodological Challenges and Guidelines"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-4877-2255","authenticated-orcid":false,"given":"Hyerim","family":"Park","sequence":"first","affiliation":[{"name":"BMW Group, Munich, Germany and University of Stuttgart, Stuttgart, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8330-147X","authenticated-orcid":false,"given":"Khanh","family":"Huynh","sequence":"additional","affiliation":[{"name":"BMW Group, Munich, Germany and LMU Munich, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4024-1645","authenticated-orcid":false,"given":"Malin","family":"Eiband","sequence":"additional","affiliation":[{"name":"BMW Group, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5739-4704","authenticated-orcid":false,"given":"Jeremy","family":"Dillmann","sequence":"additional","affiliation":[{"name":"BMW Group, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5462-8782","authenticated-orcid":false,"given":"Sven","family":"Mayer","sequence":"additional","affiliation":[{"name":"TU Dortmund University, Dortmund, Germany and Research Center Trustworthy Data Science and Security, Dortmund, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7048-9292","authenticated-orcid":false,"given":"Michael","family":"Sedlmair","sequence":"additional","affiliation":[{"name":"University of Stuttgart, Stuttgart, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,3,22]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"publisher","unstructured":"Buthayna AlMulla Maram Assi and Safwat Hassan. 2025. Understanding the Challenges and Promises of Developing Generative AI Apps: An Empirical Study. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2506.16453 (2025). 10.48550\/arXiv.2506.16453","DOI":"10.48550\/arXiv.2506.16453"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","unstructured":"Muneera Bano Didar Zowghi and Jon Whittle. 2023. Exploring qualitative research using LLMs. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2306.13298 (2023). 10.48550\/arXiv.2306.13298","DOI":"10.48550\/arXiv.2306.13298"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"publisher","unstructured":"Enrico Bertini Catherine Plaisant and Giuseppe Santucci. 2007. BELIV\u201906: beyond time and errors; novel evaluation methods for information visualization. Interactions 14 3 (2007) 59\u201360. 10.1145\/1358628.1358955","DOI":"10.1145\/1358628.1358955"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"publisher","unstructured":"Hugh Beyer and Karen Holtzblatt. 1999. Contextual design. Interactions 6 1 (Jan. 1999) 32\u201342. 10.1145\/291224.291229","DOI":"10.1145\/291224.291229"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02217-3"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"publisher","unstructured":"Virginia Braun and Victoria Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3 2 (2006) 77\u2013101. 10.1191\/1478088706qp063oa","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1037\/13620-004"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"publisher","unstructured":"Virginia Braun and Victoria Clarke. 2019. Reflecting on reflexive thematic analysis. Qualitative research in sport exercise and health 11 4 (2019) 589\u2013597. 10.1080\/2159676X.2019.1628806","DOI":"10.1080\/2159676X.2019.1628806"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"publisher","unstructured":"Sa\u0161a Brdnik Tja\u0161a Heri\u010dko and Bo\u0161tjan \u0160umak. 2022. Intelligent user interfaces and their evaluation: a systematic mapping study. Sensors 22 15 (2022) 5830. 10.3390\/s22155830","DOI":"10.3390\/s22155830"},{"key":"e_1_3_3_3_11_2","first-page":"1877","volume-title":"Advances in Neural Information Processing Systems","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems , Vol.\u00a033. Curran Associates, Inc., 1877\u20131901. https:\/\/papers.nips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","unstructured":"Daniel Buschek Malin Eiband and Heinrich Hussmann. 2022. How to Support Users in Understanding Intelligent Systems? An Analysis and Conceptual Framework of User Questions Considering User Mindsets Involvement and Knowledge Outcomes. ACM Trans. Interact. Intell. Syst. 12 4 Article 29 (Nov. 2022) 27\u00a0pages. 10.1145\/3519264","DOI":"10.1145\/3519264"},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3471391.3471416"},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/3597638.3608378"},{"key":"e_1_3_3_3_15_2","volume-title":"Designing and conducting mixed methods research","author":"Creswell John\u00a0W","year":"2017","unstructured":"John\u00a0W Creswell and Vicki L\u00a0Plano Clark. 2017. Designing and conducting mixed methods research. Sage publications."},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","unstructured":"N. Dahlb\u00e4ck A. J\u00f6nsson and L. Ahrenberg. 1993. Wizard of Oz studies \u2014 why and how. Know.-Based Syst. 6 4 (Dec. 1993) 258\u2013266. 10.1016\/0950-7051(93)90017-N","DOI":"10.1016\/0950-7051(93)90017-N"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","unstructured":"Shih-Chieh Dai Aiping Xiong and Lun-Wei Ku. 2023. LLM-in-the-loop: Leveraging large language model for thematic analysis. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.15100 (2023). 10.48550\/arXiv.2310.15100","DOI":"10.48550\/arXiv.2310.15100"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/2317956.2318020"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"publisher","unstructured":"Kathleen\u00a0M Eisenhardt. 1989. Building theories from case study research. Academy of management review 14 4 (1989) 532\u2013550. 10.2307\/258557","DOI":"10.2307\/258557"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-16-0041-8_32"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","unstructured":"Frederic Gmeiner Kenneth Holstein and Nikolas Martelaro. 2025. Prototyping Multimodal GenAI Real-Time Agents with Counterfactual Replays and Hybrid Wizard-of-Oz. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2510.06872 (2025). 10.48550\/arXiv.2510.06872","DOI":"10.48550\/arXiv.2510.06872"},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357074"},{"key":"e_1_3_3_3_23_2","unstructured":"Johanna Gustafsson. 2017. Single Case Studies vs. Multiple Case Studies: A Comparative Study. Literature review Academy of Business Engineering and Science Halmstad University Sweden."},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3663384.3663398"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","unstructured":"Adriana\u00a0Lorena Iniguez-Carrillo Laura\u00a0Sanely Gaytan-Lugo Miguel\u00a0Angel Garcia-Ruiz and Rocio Maciel-Arellano. 2021. Usability questionnaires to evaluate voice user interfaces. IEEE Latin America Transactions 19 9 (2021) 1468\u20131477. 10.1109\/TLA.2021.9468439","DOI":"10.1109\/TLA.2021.9468439"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519684"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713220"},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-67322-2_2"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/2628363.2628398"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/354401.354406"},{"key":"e_1_3_3_3_31_2","doi-asserted-by":"publisher","DOI":"10.5220\/0011383300003318"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","unstructured":"Thomas Kosch and Sebastian Feger. 2024. Risk or Chance? Large Language Models and Reproducibility in HCI Research. Interactions 31 6 (Oct. 2024) 44\u201349. 10.1145\/3695765","DOI":"10.1145\/3695765"},{"key":"e_1_3_3_3_33_2","doi-asserted-by":"publisher","unstructured":"Sebastian Krakowski. 2025. Human-AI agency in the age of generative AI. Information and Organization 35 1 (2025) 100560. 10.1016\/j.infoandorg.2025.100560","DOI":"10.1016\/j.infoandorg.2025.100560"},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"publisher","unstructured":"Heidi Lam Enrico Bertini Petra Isenberg Catherine Plaisant and Sheelagh Carpendale. 2012. Empirical studies in information visualization: Seven scenarios. IEEE transactions on visualization and computer graphics 18 9 (2012) 1520\u20131536. 10.1109\/TVCG.2011.279","DOI":"10.1109\/TVCG.2011.279"},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"publisher","DOI":"10.5555\/1841406"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706599.3721349"},{"key":"e_1_3_3_3_37_2","series-title":"(GI \u201915)","first-page":"1","volume-title":"Proceedings of the 41st Graphics Interface Conference","author":"MacKenzie I.\u00a0Scott","year":"2015","unstructured":"I.\u00a0Scott MacKenzie. 2015. User studies and usability evaluations: from research to products. In Proceedings of the 41st Graphics Interface Conference(GI \u201915). Canadian Information Processing Society, CAN, 1\u20138."},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-08-051574-8.50019-4"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"publisher","unstructured":"Tamara Munzner. 2009. A nested model for visualization design and validation. IEEE transactions on visualization and computer graphics 15 6 (2009) 921\u2013928. 10.1109\/TVCG.2009.111","DOI":"10.1109\/TVCG.2009.111"},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"publisher","unstructured":"Roderick Murray-Smith Antti Oulasvirta Andrew Howes J\u00f6rg M\u00fcller Aleksi Ikkala Miroslav Bachinski Arthur Fleig Florian Fischer and Markus Klar. 2022. What simulation can do for HCI research. Interactions 29 6 (Nov. 2022) 48\u201353. 10.1145\/3564038","DOI":"10.1145\/3564038"},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3679318.3685370"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","unstructured":"Tibor Petzoldt Hanna Bellem and Josef\u00a0F Krems. 2014. The critical tracking task: a potentially useful method to assess driver distraction?Human factors 56 4 (2014) 789\u2013808. 10.1177\/0018720813501864","DOI":"10.1177\/0018720813501864"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/989863.989880"},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-72348-6_12"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE55347.2025.00138"},{"key":"e_1_3_3_3_46_2","volume-title":"Human-Computer Interaction","author":"Preece Jenny","year":"1994","unstructured":"Jenny Preece, Yvonne Rogers, Helen Sharp, David Benyon, Simon Holland, and Tom Carey. 1994. Human-Computer Interaction. Addison-Wesley Longman Ltd., GBR."},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","unstructured":"Anna Ravera and Cristina Gena. 2025. On the usability of generative AI: Human generative AI. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2502.17714 (2025). 10.48550\/arXiv.2502.17714","DOI":"10.48550\/arXiv.2502.17714"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/3635636.3656184"},{"key":"e_1_3_3_3_49_2","doi-asserted-by":"publisher","unstructured":"Laurel\u00a0D. Riek. 2012. Wizard of Oz studies in HRI: a systematic review and new reporting guidelines. J. Hum.-Robot Interact. 1 1 (July 2012) 119\u2013136. 10.5898\/JHRI.1.1.Riek","DOI":"10.5898\/JHRI.1.1.Riek"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"publisher","unstructured":"Alexander Rind. 2011. Some Whys and Hows of Experiments in Human\u2013Computer Interaction. Foundations and Trends\u00ae in Human-Computer Interaction 5 (Jan. 2011) 299\u2013373. 10.1561\/1100000043","DOI":"10.1561\/1100000043"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02220-3_4"},{"key":"e_1_3_3_3_52_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02220-3"},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","unstructured":"Yao Rong Tobias Leemann Thai-Trang Nguyen Lisa Fiedler Peizhu Qian Vaibhav Unhelkar Tina Seidel Gjergji Kasneci and Enkelejda Kasneci. 2024. Towards Human-Centered Explainable AI: A Survey of User Studies for Model Explanations. IEEE Transactions on Pattern Analysis and Machine Intelligence 46 4 (April 2024) 2104\u20132122. 10.1109\/TPAMI.2023.3331846","DOI":"10.1109\/TPAMI.2023.3331846"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3445022"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"publisher","unstructured":"Michael Sedlmair Miriah Meyer and Tamara Munzner. 2012. Design study methodology: Reflections from the trenches and the stacks. IEEE transactions on visualization and computer graphics 18 12 (2012) 2431\u20132440. 10.1109\/TVCG.2012.213","DOI":"10.1109\/TVCG.2012.213"},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","unstructured":"Jingyu Shi Rahul Jain Hyungjun Doh Ryo Suzuki and Karthik Ramani. 2023. An HCI-centric survey and taxonomy of human-generative-AI interactions. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.07127 (2023). 10.48550\/arXiv.2310.07127","DOI":"10.48550\/arXiv.2310.07127"},{"key":"e_1_3_3_3_57_2","doi-asserted-by":"publisher","DOI":"10.1145\/3659677.3659825"},{"key":"e_1_3_3_3_58_2","doi-asserted-by":"publisher","unstructured":"Auste Simkute Lev Tankelevitch Viktor Kewenig Ava\u00a0Elizabeth Scott Abigail Sellen and Sean Rintel. 2025. Ironies of generative AI: understanding and mitigating productivity loss in Human-AI interaction. International Journal of Human\u2013Computer Interaction 41 5 (2025) 2898\u20132919. 10.1080\/10447318.2024.2405782","DOI":"10.1080\/10447318.2024.2405782"},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"publisher","unstructured":"Hannu Simonen Atte Kiviniemi and Jonas Oppenlaender. 2025. An Initial Exploration of Default Images in Text-to-Image Generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2505.09166 (2025). 10.48550\/arXiv.2505.09166","DOI":"10.48550\/arXiv.2505.09166"},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"publisher","unstructured":"Sonali\u00a0Uttam Singh and Akbar\u00a0Siami Namin. 2025. A survey on chatbots and large language models: Testing and evaluation techniques. Natural Language Processing Journal (2025) 100128. 10.1016\/j.nlp.2025.100128","DOI":"10.1016\/j.nlp.2025.100128"},{"key":"e_1_3_3_3_61_2","doi-asserted-by":"publisher","DOI":"10.1145\/1514095.1514115"},{"key":"e_1_3_3_3_62_2","doi-asserted-by":"publisher","DOI":"10.1177\/1071181320641004"},{"key":"e_1_3_3_3_63_2","doi-asserted-by":"publisher","unstructured":"Elias Storms Oscar Alvarado and Luciana Monteiro-Krebs. 2022. \u2019Transparency is Meant for Control\u2019 and Vice Versa: Learning from Co-designing and Evaluating Algorithmic News Recommenders. Proc. ACM Hum.-Comput. Interact. 6 CSCW2 Article 405 (Nov. 2022) 24\u00a0pages. 10.1145\/3555130","DOI":"10.1145\/3555130"},{"key":"e_1_3_3_3_64_2","doi-asserted-by":"publisher","DOI":"10.1145\/3708036.3708042"},{"key":"e_1_3_3_3_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3711542.3711583"},{"key":"e_1_3_3_3_66_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445497"},{"key":"e_1_3_3_3_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/2639189.2639220"},{"key":"e_1_3_3_3_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377500"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"publisher","DOI":"10.1145\/67449.67500"},{"key":"e_1_3_3_3_70_2","doi-asserted-by":"publisher","unstructured":"Alan Woolrych Kasper Hornb\u00e6k Erik Fr\u00f8kj\u00e6r and Gilbert Cockton. 2011. Ingredients and meals rather than recipes: A proposal for research that does not treat usability evaluation methods as indivisible wholes. International Journal of Human-Computer Interaction 27 10 (2011) 940\u2013970. 10.1080\/10447318.2011.555314","DOI":"10.1080\/10447318.2011.555314"},{"key":"e_1_3_3_3_71_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3650732"},{"key":"e_1_3_3_3_72_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"e_1_3_3_3_73_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713337"},{"key":"e_1_3_3_3_74_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713347"},{"key":"e_1_3_3_3_75_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_76_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3714045"}],"event":{"name":"IUI '26: 31st International Conference on Intelligent User Interfaces","location":"Paphos Cyprus","acronym":"IUI '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 31st International Conference on Intelligent User Interfaces"],"original-title":[],"deposited":{"date-parts":[[2026,3,14]],"date-time":"2026-03-14T13:02:33Z","timestamp":1773493353000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3742413.3789065"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3,22]]},"references-count":75,"alternative-id":["10.1145\/3742413.3789065","10.1145\/3742413"],"URL":"https:\/\/doi.org\/10.1145\/3742413.3789065","relation":{},"subject":[],"published":{"date-parts":[[2026,3,22]]},"assertion":[{"value":"2026-03-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}