{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T12:21:50Z","timestamp":1776082910598,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":81,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,11]],"date-time":"2024-08-11T00:00:00Z","timestamp":1723334400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"III.Lab (Intelligent Interfaces Innovation) funded by AIT Austrian Institute of Technology GmbH, University of Salzburg and Land Salzburg","award":["20102\/F2300416-KZP"],"award-info":[{"award-number":["20102\/F2300416-KZP"]}]},{"name":"Austrian Research Promotion Agency","award":["879324"],"award-info":[{"award-number":["879324"]}]},{"DOI":"10.13039\/501100006374","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","award":["833672"],"award-info":[{"award-number":["833672"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,11]]},"DOI":"10.1145\/3666094.3666103","type":"proceedings-article","created":{"date-parts":[[2024,8,29]],"date-time":"2024-08-29T18:21:01Z","timestamp":1724955661000},"page":"133-144","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Talk the Talk, Walk the Walk: A Case Study on Systematic Evaluation in Participatory Design"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9231-2575","authenticated-orcid":false,"given":"Quynh","family":"Nguyen","sequence":"first","affiliation":[{"name":"Center for Technology Experience, Austrian Institute of Technology GmbH, Austria and Department for Artificial Intelligence and Human Interfaces, University of Salzburg, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1833-2762","authenticated-orcid":false,"given":"Emma","family":"Jaspaert","sequence":"additional","affiliation":[{"name":"Criminal Law, Tilburg University, Netherlands and Leuven Institute of Criminology, KU Leuven, Belgium"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-3592-0494","authenticated-orcid":false,"given":"Birgit","family":"Harthum","sequence":"additional","affiliation":[{"name":"Research Projects & Consulting, Usecon GmbH, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,8,29]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"121","article-title":"Creating Protocols for Trustworthiness in Qualitative Research","volume":"23","author":"Amankwaa Linda","year":"2016","unstructured":"Linda Amankwaa. 2016. Creating Protocols for Trustworthiness in Qualitative Research. Journal of cultural diversity 23, 3 (2016), 121\u2013127. https:\/\/pubmed.ncbi.nlm.nih.gov\/29694754\/","journal-title":"Journal of cultural diversity"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710882.2015.1081246"},{"key":"e_1_3_2_1_3_1","volume-title":"Criteria for evaluating qualitative research. Human Resource Development Quarterly","author":"Anderson Valerie","year":"2017","unstructured":"Valerie Anderson. 2017. Criteria for evaluating qualitative research. Human Resource Development Quarterly (2017), 1\u20139. https:\/\/pure.port.ac.uk\/ws\/files\/6930813\/ANDERSONv_2017_cright_HRDQ_Criteria_for_Evaluating_Qualitative_Research.pdf"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3177794"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijmedinf.2021.104394"},{"key":"e_1_3_2_1_6_1","unstructured":"Kathy Baxter Catherine Courage and Kelly Caine. 2015. Understanding your users: a practical guide to user research methods. Morgan Kaufmann."},{"key":"e_1_3_2_1_7_1","volume-title":"A UTOPIAN experience: On design of powerful computer-based tools for skilled graphic workers","author":"B\u00f8dker Susanne","year":"1987","unstructured":"Susanne B\u00f8dker, Pelle Ehn, John Kammersgaard, Morten Kyng, and Yngve Sundblad. 1987. A UTOPIAN experience: On design of powerful computer-based tools for skilled graphic workers. Computers and Democracy G. Bjerknes, P. Ehn, and M. Knyg (Eds.). Avebury Pub. England (1987), 25."},{"key":"e_1_3_2_1_8_1","volume-title":"Design at work","author":"B\u00f8dker Susanne","unstructured":"Susanne B\u00f8dker and Kaj Gr\u00f8nb\u00e6k. 1991. Design in action: From prototyping by demonstration to cooperative prototyping. In Design at work. CRC Press, 197\u2013218."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/572020.572023"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940299.2940303"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3210604.3210638"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2347635.2347642"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2662155.2662192"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10606-016-9259-4"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-10-5251-4_103"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1177\/14767503030012002"},{"key":"e_1_3_2_1_17_1","volume-title":"The Oxford Handbook of Qualitative Research","author":"Cho Jeasik","unstructured":"Jeasik Cho and Allen Trent. 2014. Evaluating Qualitative Research. In The Oxford Handbook of Qualitative Research, Patricia Leavy (Ed.). Oxford University Press, Chapter\u00a032, 677\u2013696."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/1098214013478142"},{"key":"e_1_3_2_1_19_1","volume-title":"Thematic analysis: a practical guide. Thematic Analysis","author":"Clarke Victoria","year":"2021","unstructured":"Victoria Clarke and Virginia Braun. 2021. Thematic analysis: a practical guide. Thematic Analysis (2021)."},{"key":"e_1_3_2_1_20_1","volume-title":"Participation: The new tyranny?Zed books.","author":"Cooke Bill","year":"2001","unstructured":"Bill Cooke and Uma Kothari. 2001. Participation: The new tyranny?Zed books."},{"key":"e_1_3_2_1_21_1","volume-title":"Basics of qualitative research","author":"Corbin Juliet","unstructured":"Juliet Corbin and Anselm Strauss. 2008. Basics of qualitative research. 3rd ed. Thousand Oaks.","edition":"3"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.21606\/drs.2018.679"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.3102\/01623737014004397"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1002\/ev.1114"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2003.11.002"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710882.2018.1540641"},{"key":"e_1_3_2_1_27_1","volume-title":"ESEC \u201989, C.\u00a0Ghezzi and J.\u00a0A","author":"Floyd Christiane","unstructured":"Christiane Floyd, Fanny-Michaela Reisin, and Gerhard Schmidt. 1989. STEPS to software development with users. In ESEC \u201989, C.\u00a0Ghezzi and J.\u00a0A. McDermid (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 48\u201364."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"Christopher Frauenberger Judith Good Geraldine Fitzpatrick and Ole\u00a0Sejer Iversen. 2015. In Pursuit of Rigour and Accountability in Participatory Design. International journal of human-computer studies 74 (2015) 93\u2013106. https:\/\/doi.org\/10.1016\/j.ijhcs.2014.09.004","DOI":"10.1016\/j.ijhcs.2014.09.004"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5590\/JOSC.2018.10.1.02"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661435.2661451"},{"key":"e_1_3_2_1_31_1","volume-title":"What is a case study and what is it good for?American political science review 98, 2","author":"Gerring John","year":"2004","unstructured":"John Gerring. 2004. What is a case study and what is it good for?American political science review 98, 2 (2004), 341\u2013354."},{"key":"e_1_3_2_1_32_1","unstructured":"Davydd\u00a0J Greenwood and Morten Levin. 2006. Introduction to action research: Social research for social change. SAGE publications. 66\u201367 pages."},{"key":"e_1_3_2_1_33_1","volume-title":"Toward a Cooperative Experimental System Development Approach","author":"Gr\u00f8nb\u00e6k Kaj","unstructured":"Kaj Gr\u00f8nb\u00e6k, Morten Kyng, and Preben Mogensen. 1997. Toward a Cooperative Experimental System Development Approach. MIT Press, Cambridge, MA, USA, 201\u2013238."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02766777"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710880902920960"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2014.09.003"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2598784.2611381"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/1900441.1900451"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710882.2012.672575"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1207\/s15327051hci1302_3"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3384772.3385145"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319376"},{"key":"e_1_3_2_1_43_1","first-page":"15","article-title":"Design choices framework for co-creation projects","volume":"12","author":"Lee Jung\u00a0Joo","year":"2018","unstructured":"Jung\u00a0Joo Lee, Miia Jaatinen, Anna Salmi, Tuuli Mattelm\u00e4ki, Riitta Smeds, and Mari Holopainen. 2018. Design choices framework for co-creation projects. International Journal of Design 12, 2 (2018), 15\u201331.","journal-title":"International Journal of Design"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2347635.2347645"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1080\/15614263.2023.2176307"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1011870.1011872"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1177\/1558689807302811"},{"key":"e_1_3_2_1_48_1","unstructured":"Matthew\u00a0B Miles and A\u00a0Michael Huberman. 1994. Qualitative data analysis: An expanded sourcebook. sage."},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 59th Annual Meeting of the ISSS-2015 Berlin, Germany, Vol.\u00a01.","author":"Mossel Annette","year":"2015","unstructured":"Annette Mossel, Andreas Peer, Johannes G\u00f6llner, and Hannes Kaufmann. 2015. Requirements analysis on a virtual reality training system for CBRN crisis preparedness. In Proceedings of the 59th Annual Meeting of the ISSS-2015 Berlin, Germany, Vol.\u00a01."},{"key":"e_1_3_2_1_50_1","volume-title":"12th International Multidisciplinary Modeling & Simulation Multiconference. DIME University of Genoa, DIMEH University of Calabria.","author":"Mossel Annette","year":"2015","unstructured":"Annette Mossel, Andreas Peer, Johannes G\u00f6llner, and Hannes Kaufmann. 2015. Towards an immersive virtual reality training system for CBRN disaster preparedness. In The 12th International Multidisciplinary Modeling & Simulation Multiconference. DIME University of Genoa, DIMEH University of Calabria."},{"key":"e_1_3_2_1_51_1","volume-title":"The Human\u2013Computer Interaction Handbook","author":"Muller J","unstructured":"Michael\u00a0J Muller and Allison Druin. 2012. Participatory design: the third space in human\u2013computer interaction. In The Human\u2013Computer Interaction Handbook. CRC Press, 1125\u20131153."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3537797.3537828"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3656650.3656686"},{"key":"e_1_3_2_1_54_1","volume-title":"Stress Out: Translating Real-World Stressors into Audio-Visual Stress Cues in VR for Police Training. In Human-Computer Interaction \u2013 INTERACT","author":"Nguyen Quynh","year":"2021","unstructured":"Quynh Nguyen, Emma Jaspaert, Markus Murtinger, Helmut Schrom-Feiertag, Sebastian Egger-Lampl, and Manfred Tscheligi. 2021. Stress Out: Translating Real-World Stressors into Audio-Visual Stress Cues in VR for Police Training. In Human-Computer Interaction \u2013 INTERACT 2021, Carmelo Ardito, Rosa Lanzilotti, Alessio Malizia, Helen Petrie, Antonio Piccinno, Giuseppe Desolda, and Kori Inkpen (Eds.). Springer International Publishing, Cham, 551\u2013561."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","unstructured":"B.\u00a0A. Nosek G. Alter G.\u00a0C. Banks D. Borsboom S.\u00a0D. Bowman S.\u00a0J. Breckler S. Buck C.\u00a0D. Chambers G. Chin G. Christensen M. Contestabile A. Dafoe E. Eich J. Freese R. Glennerster D. Goroff D.\u00a0P. Green B. Hesse M. Humphreys J. Ishiyama D. Karlan A. Kraut A. Lupia P. Mabry T. Madon N. Malhotra E. Mayo-Wilson M. McNutt E. Miguel E.\u00a0Levy Paluck U. Simonsohn C. Soderberg B.\u00a0A. Spellman J. Turitto G. VandenBos S. Vazire E.\u00a0J. Wagenmakers R. Wilson and T. Yarkoni. 2015. Promoting an open research culture. Science 348 6242 (2015) 1422\u20131425. https:\/\/doi.org\/10.1126\/science.aab2374","DOI":"10.1126\/science.aab2374"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3384772.3385141"},{"key":"e_1_3_2_1_57_1","volume-title":"14th International Conference on Modeling and Applied Simulation, MAS","author":"Pallavicini Federica","year":"2015","unstructured":"Federica Pallavicini, Nicola Toniazzi, Luca Argenton, Luciana Aceti, Fabrizia Mantovani, 2015. Developing effective virtual reality training for military forces and emergency operators: from technology to human factors. In 14th International Conference on Modeling and Applied Simulation, MAS 2015. Dime University of Genoa, 206\u2013210."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.3390\/mti7090088"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710880701875068"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.3390\/architecture2030029"},{"key":"e_1_3_2_1_61_1","volume-title":"The SAGE handbook of qualitative research","author":"Schwandt A","unstructured":"Thomas\u00a0A Schwandt and Emily\u00a0F Gates. 2017. Case study methodology. In The SAGE handbook of qualitative research. SAGE Publishing."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.3233\/EFI-2004-22201"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1162\/DESI_a_00158"},{"key":"e_1_3_2_1_64_1","volume-title":"Effects-Driven IT Improvement: Pursuing Local Post-Implementation Opportunities. Scandinavian Journal of Information Systems","author":"Simonsen Jesper","year":"2022","unstructured":"Jesper Simonsen and Morten Hertzum. 2022. Effects-Driven IT Improvement: Pursuing Local Post-Implementation Opportunities. Scandinavian Journal of Information Systems (2022)."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","unstructured":"Jesper Simonsen and Toni Robertson. 2012. Routledge international handbook of participatory design. https:\/\/doi.org\/10.4324\/9780203108543","DOI":"10.4324\/9780203108543"},{"key":"e_1_3_2_1_66_1","first-page":"163","article-title":"The Methodology of Participatory Design","volume":"52","author":"Spinuzzi Clay","year":"2005","unstructured":"Clay Spinuzzi. 2005. The Methodology of Participatory Design. Technical Communication 52, 2 (2005), 163\u2013174. https:\/\/www.ingentaconnect.com\/content\/stc\/tc\/2005\/00000052\/00000002\/art00005","journal-title":"Technical Communication"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1162\/DESI_a_00125"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3385010.3385026"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3210586.3210601"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.3168\/jds.S0022-0302(94)77044-2"},{"key":"e_1_3_2_1_71_1","volume-title":"Design Anthropology","author":"Tunstall Elizabeth\u00a0Dori","unstructured":"Elizabeth\u00a0Dori Tunstall. 2020. Decolonizing design innovation: Design anthropology, critical anthropology, and indigenous knowledge. In Design Anthropology. Routledge, 232\u2013250."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1017\/dsj.2022.24"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470716"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710880500135987"},{"key":"e_1_3_2_1_75_1","volume-title":"Participatory-Action Research, Agency, and Representation","author":"Williams T","year":"2004","unstructured":"Bronwyn\u00a0T Williams and Mary Brydon-Miller. 2004. Changing directions. Ethnography Unbound from Theory Shock to Critical Praxis, Participatory-Action Research, Agency, and Representation (2004), 241\u2013258."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3532106.3533539"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10606-022-09457-0"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710882.2019.1605383"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1080\/15710882.2019.1605383"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.3390\/mti7020014"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-42293-5_16"}],"event":{"name":"PDC '24: Participatory Design Conference 2024","location":"Sibu Malaysia","acronym":"PDC '24"},"container-title":["Participatory Design Conference 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3666094.3666103","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3666094.3666103","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,29]],"date-time":"2025-08-29T16:40:33Z","timestamp":1756485633000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3666094.3666103"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,11]]},"references-count":81,"alternative-id":["10.1145\/3666094.3666103","10.1145\/3666094"],"URL":"https:\/\/doi.org\/10.1145\/3666094.3666103","relation":{},"subject":[],"published":{"date-parts":[[2024,8,11]]},"assertion":[{"value":"2024-08-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}