{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T23:08:46Z","timestamp":1763507326392,"version":"3.45.0"},"publisher-location":"Cham","reference-count":30,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030902377"},{"type":"electronic","value":"9783030902384"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-90238-4_3","type":"book-chapter","created":{"date-parts":[[2021,11,19]],"date-time":"2021-11-19T07:11:14Z","timestamp":1637305874000},"page":"24-39","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["The Crowd Thinks Aloud: Crowdsourcing Usability Testing with\u00a0the\u00a0Thinking Aloud Method"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5037-5279","authenticated-orcid":false,"given":"Edwin","family":"Gamboa","sequence":"first","affiliation":[]},{"given":"Rahul","family":"Galda","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4586-8834","authenticated-orcid":false,"given":"Cindy","family":"Mayas","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1359-363X","authenticated-orcid":false,"given":"Matthias","family":"Hirth","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,11,20]]},"reference":[{"issue":"3","key":"3_CR1","first-page":"111","volume":"12","author":"O Alhadreti","year":"2017","unstructured":"Alhadreti, O., Mayhew, P.: To intervene or not to intervene: an investigation of three think-aloud protocols in usability testing. J. Usability Stud. 12(3), 111\u2013132 (2017)","journal-title":"J. Usability Stud."},{"issue":"3","key":"3_CR2","doi-asserted-by":"publisher","first-page":"800","DOI":"10.3758\/s13428-011-0081-0","volume":"43","author":"TS Behrend","year":"2011","unstructured":"Behrend, T.S., Sharek, D.J., Meade, A.W., Wiebe, E.N.: The viability of crowdsourcing for survey research. Behav. Res. Methods 43(3), 800\u2013813 (2011). https:\/\/doi.org\/10.3758\/s13428-011-0081-0","journal-title":"Behav. Res. Methods"},{"issue":"1","key":"3_CR3","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1016\/j.jsb.2018.02.006","volume":"203","author":"J Bruggemann","year":"2018","unstructured":"Bruggemann, J., Lander, G.C., Su, A.I.: Exploring applications of crowdsourcing to cryo-EM. J. Struct. Biol. 203(1), 37\u201345 (2018). https:\/\/doi.org\/10.1016\/j.jsb.2018.02.006","journal-title":"J. Struct. Biol."},{"issue":"17","key":"3_CR4","doi-asserted-by":"publisher","first-page":"1285","DOI":"10.1177\/154193129003401723","volume":"34","author":"S Denning","year":"1990","unstructured":"Denning, S., Hoiem, D., Simpson, M., Sullivan, K.: The value of thinking-aloud protocols in industry: a case study at microsoft corporation. Proc. Hum. Factors Soc. Ann. Meet. 34(17), 1285\u20131289 (1990). https:\/\/doi.org\/10.1177\/154193129003401723","journal-title":"Proc. Hum. Factors Soc. Ann. Meet."},{"key":"3_CR5","doi-asserted-by":"publisher","unstructured":"Egger-Lampl, S., et al.: Crowdsourcing quality of experience experiments. In: Archambault, D., Purchase, H., Ho\u00dffeld, T. (eds.) Evaluation in the Crowd. Crowdsourcing and Human-Centered Experiments. Evaluation in the Crowd. Crowdsourcing and Human-Centered Experiments, LNCS, vol. 10264, pp. 154\u2013190. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-66435-4_7","DOI":"10.1007\/978-3-319-66435-4_7"},{"key":"3_CR6","volume-title":"Protocol Analysis: Verbal Reports as Data","author":"KA Ericsson","year":"1984","unstructured":"Ericsson, K.A., Simon, H.A.: Protocol Analysis: Verbal Reports as Data. The MIT Press, Cambridge (1984)"},{"key":"3_CR7","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1177\/0165551512437638","volume":"38","author":"EL Estell\u00e9s-Arolas","year":"2012","unstructured":"Estell\u00e9s-Arolas, E.L., Guevara, F.G., Towards an integrated crowdsourcing definition: Towards an integrated crowdsourcing definition. J. Inf. Sci. 38, 189\u2013200 (2012). https:\/\/doi.org\/10.1177\/0165551512437638","journal-title":"J. Inf. Sci."},{"key":"3_CR8","unstructured":"Fan, M., Shi, S., Truong, K.N.: Practices and challenges of using think-aloud protocols in industry: an international survey. J. Usability Stud. 15(2) (2020)"},{"key":"3_CR9","doi-asserted-by":"publisher","unstructured":"Haak, M.J., Jong, M.D., Schellens, P.J.: Evaluating municipal websites: a methodological comparison of three think-aloud variants. Gov. Inf. Qual. 26(1) (2009). https:\/\/doi.org\/10.1016\/j.giq.2007.11.003","DOI":"10.1016\/j.giq.2007.11.003"},{"key":"3_CR10","doi-asserted-by":"publisher","unstructured":"Hirth, M., Ho\u00dffeld, T., Tran-Gia, P.: Anatomy of a crowdsourcing platform - using the example of Microworkers.com. In: International Conference on Innovative Mobile and Internet Services in Ubiquitous Computing (2011). https:\/\/doi.org\/10.1109\/IMIS.2011.89","DOI":"10.1109\/IMIS.2011.89"},{"issue":"2","key":"3_CR11","doi-asserted-by":"publisher","first-page":"541","DOI":"10.1109\/TMM.2013.2291663","volume":"16","author":"T Hossfeld","year":"2014","unstructured":"Hossfeld, T., et al.: Best practices for QOE crowdtesting: QOE assessment with crowdsourcing. IEEE Trans. Multimedia 16(2), 541\u2013558 (2014). https:\/\/doi.org\/10.1109\/TMM.2013.2291663","journal-title":"IEEE Trans. Multimedia"},{"key":"3_CR12","doi-asserted-by":"publisher","unstructured":"Kittur, A., Chi, E.H., Suh, B.: Crowdsourcing user studies with mechanical Turk. In: Conference on Human Factors in Computing Systems (2008). https:\/\/doi.org\/10.1145\/1357054.1357127","DOI":"10.1145\/1357054.1357127"},{"key":"3_CR13","unstructured":"Krug, S.: Rocket Surgery Made Easy: The Do-It-Yourself Guide to Finding and Fixing Usability Problems. New Riders (2009)"},{"key":"3_CR14","volume-title":"Don\u2019t Make Me Think","author":"S Krug","year":"2014","unstructured":"Krug, S.: Don\u2019t Make Me Think. Revisited - A Common Sense Approach to Web Usability, New Riders (2014)"},{"key":"3_CR15","volume-title":"Using the \u201cThinking Aloud\u201d Method in Cognitive Interface Design","author":"C Lewis","year":"1982","unstructured":"Lewis, C.: Using the \u201cThinking Aloud\u2019\u2019 Method in Cognitive Interface Design. IBM TJ Watson Research Center Yorktown Heights, NY (1982)"},{"key":"3_CR16","doi-asserted-by":"publisher","unstructured":"Liu, D., Bias, R.G., Lease, M., Kuipers, R.: Crowdsourcing for usability testing. In: Proceedings of the American Society for Information Science and Technology 49(1) (2012). https:\/\/doi.org\/10.1002\/meet.14504901100","DOI":"10.1002\/meet.14504901100"},{"issue":"17","key":"3_CR17","doi-asserted-by":"publisher","first-page":"438","DOI":"10.1007\/978-3-319-10470-6_55","volume":"2014","author":"L Maier-Hein","year":"2014","unstructured":"Maier-Hein, L., et al.: Can masses of non-experts train highly accurate image classifiers? a crowdsourcing approach to instrument segmentation in laparoscopic images. Med. Image Comput. Comput.-Assist. Interv. - MICCAI 2014(17), 438\u2013445 (2014). https:\/\/doi.org\/10.1007\/978-3-319-10470-6_55","journal-title":"Med. Image Comput. Comput.-Assist. Interv. - MICCAI"},{"key":"3_CR18","doi-asserted-by":"publisher","unstructured":"Meier, F.: Crowdsourcing als Rekrutierungsstrategie im Asynchronen Remote-Usability-Test. Information-Wissenschaft und Praxis, 63(5) (2012). https:\/\/doi.org\/10.1515\/iwp-2012-0063","DOI":"10.1515\/iwp-2012-0063"},{"key":"3_CR19","doi-asserted-by":"publisher","unstructured":"Nebeling, M., Speicher, M., Norrie, M.C.: CrowdStudy: general toolkit for Crowdsourced evaluation of web interfaces. In: SIGCHI Symposium on Engineering Interactive Computing Systems (2013). https:\/\/doi.org\/10.1145\/2494603.2480303","DOI":"10.1145\/2494603.2480303"},{"key":"3_CR20","doi-asserted-by":"crossref","unstructured":"Nielsen, J.: Usability Engineering. Elsevier Science (1994)","DOI":"10.1016\/B978-0-08-052029-2.50007-3"},{"key":"3_CR21","doi-asserted-by":"crossref","unstructured":"Nielsen, J.: Estimating the number of subjects needed for a thinking aloud test. Int. J. Hum. - Comput. Stud. 41(3) (1994)","DOI":"10.1006\/ijhc.1994.1065"},{"key":"3_CR22","doi-asserted-by":"publisher","unstructured":"Nielsen, L., Chavan, S.: Differences in task descriptions in the think aloud test. In: Aykin, N. (ed.) Usability and Internationalization. HCI and Culture, UI-HCII 2007. LNCS, vol. 4559, pp. 174\u2013180. Springer, Heidelberg (2007). https:\/\/doi.org\/10.1007\/978-3-540-73287-7_22","DOI":"10.1007\/978-3-540-73287-7_22"},{"key":"3_CR23","doi-asserted-by":"crossref","unstructured":"Rosenthal, R.: Meta-Analytic Procedures for Social Research. SAGE Publications, Inc. (1991)","DOI":"10.4135\/9781412984997"},{"key":"3_CR24","volume-title":"Handbook of Usability Testing: How to Plan, Design, and Conduct Effective Tests","author":"J Rubin","year":"2011","unstructured":"Rubin, J., Chisnell, D., Spool, J.: Handbook of Usability Testing: How to Plan, Design, and Conduct Effective Tests. Wiley, Hoboken (2011)"},{"key":"3_CR25","doi-asserted-by":"publisher","unstructured":"Schneider, H., Frison, K., Wagner, J., Butz, A.: CrowdUX: a case for using widespread and lightweight tools in the quest for UX. In: Conference on Designing Interactive Systems (2016). https:\/\/doi.org\/10.1145\/2901790.2901814","DOI":"10.1145\/2901790.2901814"},{"key":"3_CR26","unstructured":"Skov, M.B., Stage, J.: Supporting problem identification in usability evaluations. In: Australia Conference on Computer-Human Interaction (2005)"},{"key":"3_CR27","unstructured":"Sova, D.H., Nielsen, J.: 234 Tips and Tricks for Recruiting Users as Participants in Usability Studies. Nielsen N Group (2003)"},{"key":"3_CR28","doi-asserted-by":"publisher","unstructured":"Thompson, K.E., Rozanski, E.P., Haake, A.R.: Here, there, anywhere: remote usability testing that works. In: Conference on Information Technology Education (2004). https:\/\/doi.org\/10.1145\/1029533.1029567","DOI":"10.1145\/1029533.1029567"},{"key":"3_CR29","doi-asserted-by":"crossref","unstructured":"Vermeeren, A.P.O.S., den Bouwmeester, K., Aasman, J., Ridder, H., de Ridder, H.: DEVAN: a tool for detailed video analysis of user test aata. Behav. IT 21(6) (2002)","DOI":"10.1080\/0144929021000051714"},{"key":"3_CR30","doi-asserted-by":"publisher","unstructured":"Yuhui, W., Tian, L., Xinxiong, L.: Reliability of perceived usability assessment via crowdsourcing platform: retrospective analysis and novel feedback quality inspection method. Int. J. Hum.-Comput. Int. 36(11) (2020). https:\/\/doi.org\/10.1080\/10447318.2019.1709339","DOI":"10.1080\/10447318.2019.1709339"}],"container-title":["Lecture Notes in Computer Science","HCI International 2021 - Late Breaking Papers: Design and User Experience"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-90238-4_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T23:05:01Z","timestamp":1763507101000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-90238-4_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030902377","9783030902384"],"references-count":30,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-90238-4_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"20 November 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"HCII","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Human-Computer Interaction","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24 July 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 July 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"hcii2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/2021.hci.international\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}