{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,3]],"date-time":"2025-08-03T22:51:48Z","timestamp":1754261508148,"version":"3.28.0"},"reference-count":113,"publisher":"Informa UK Limited","issue":"22","content-domain":{"domain":["www.tandfonline.com"],"crossmark-restriction":true},"short-container-title":["International Journal of Human\u2013Computer Interaction"],"published-print":{"date-parts":[[2024,11,16]]},"DOI":"10.1080\/10447318.2023.2263694","type":"journal-article","created":{"date-parts":[[2023,10,9]],"date-time":"2023-10-09T11:50:09Z","timestamp":1696852209000},"page":"7297-7313","update-policy":"http:\/\/dx.doi.org\/10.1080\/tandf_crossmark_01","source":"Crossref","is-referenced-by-count":4,"title":["A New Method for Identifying Low-Quality Data in Perceived Usability Crowdsourcing Tests: Differences in Questionnaire Scores"],"prefix":"10.1080","volume":"40","author":[{"given":"Yuhui","family":"Wang","sequence":"first","affiliation":[{"name":"Department of Industrial Design, School of Architecture and Art, Central South University, Changsha, China"}]},{"given":"Xuan","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Industrial Design, School of Architecture and Art, Central South University, Changsha, China"}]},{"given":"Xuan","family":"Zhou","sequence":"additional","affiliation":[{"name":"Department of Industrial Design, School of Architecture and Art, Central South University, Changsha, China"}]}],"member":"301","published-online":{"date-parts":[[2023,10,9]]},"reference":[{"key":"e_1_3_2_2_1","volume-title":"Measuring the user experience","author":"Albert W.","year":"2013","unstructured":"Albert, W., & Tullis, T. (2013). Measuring the user experience. China Machine Press."},{"key":"e_1_3_2_3_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2020.1809152"},{"key":"e_1_3_2_4_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2021.1938391"},{"key":"e_1_3_2_5_1","volume-title":"Paper Presented at the Science & Information Conference","author":"Alharbi A.","year":"2015","unstructured":"Alharbi, A., & Mayhew, P. (2015). Users\u2019 performance in lab and non-lab enviornments through online usability testing: A case of evaluating the usability of digital academic libraries\u2019 websites [Paper presentation]. Paper Presented at the Science & Information Conference."},{"key":"e_1_3_2_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2013.20"},{"key":"e_1_3_2_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106363"},{"key":"e_1_3_2_8_1","doi-asserted-by":"publisher","DOI":"10.14569\/IJACSA.2017.080605"},{"key":"e_1_3_2_9_1","volume-title":"Psychological testing","author":"Anastasi A.","year":"1988","unstructured":"Anastasi, A. (1988). Psychological testing (5th ed.). Macmillan.","edition":"5"},{"key":"e_1_3_2_10_1","doi-asserted-by":"crossref","unstructured":"Andersen S. B. Rasmussen C. K. & Fr\u00f8kj\u00e6r E. (2017). Bringing content understanding into usability testing in complex application domains\u2014a case study in eHealth. Design User Experience and Usability: Theory Methodology and Management: 6th International Conference DUXU 2017 Vancouver BC Canada July 9\u201314 (pp. 327\u2013341). Springer International Publishing.","DOI":"10.1007\/978-3-319-58634-2_25"},{"key":"e_1_3_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240838"},{"key":"e_1_3_2_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-49340-4_24"},{"key":"e_1_3_2_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.intell.2017.03.003"},{"issue":"1","key":"e_1_3_2_14_1","first-page":"15","article-title":"Standardized usability questionnaires: Features and quality focus","volume":"6","author":"Assila A.","year":"2016","unstructured":"Assila, A., de Oliveira, K., & Ezzedine, H. (2016). Standardized usability questionnaires: Features and quality focus. Electronic Journal of Computer Science and Information Technology, 6(1), 15\u201331.","journal-title":"Electronic Journal of Computer Science and Information Technology,"},{"key":"e_1_3_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1135777.1135811"},{"key":"e_1_3_2_16_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447310802205776"},{"key":"e_1_3_2_17_1","doi-asserted-by":"publisher","DOI":"10.1177\/1525822X12443097"},{"key":"e_1_3_2_18_1","doi-asserted-by":"publisher","DOI":"10.15288\/jsad.2012.73.834"},{"key":"e_1_3_2_19_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2020.102575"},{"key":"e_1_3_2_20_1","doi-asserted-by":"publisher","DOI":"10.1080\/00223980.1989.10542966"},{"issue":"3","key":"e_1_3_2_21_1","first-page":"89","article-title":"Re-assessing the usability metric for user experience (UMUX) scale","volume":"11","author":"Berkman M. I.","year":"2016","unstructured":"Berkman, M. I., & Karahoca, D. (2016). Re-assessing the usability metric for user experience (UMUX) scale. Journal of Usability Studies, 11(3), 89\u2013109.","journal-title":"Journal of Usability Studies,"},{"key":"e_1_3_2_22_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2015.1064648"},{"issue":"194","key":"e_1_3_2_23_1","first-page":"4","article-title":"SUS-A quick and dirty usability scale","volume":"189","author":"Brooke J.","year":"1996","unstructured":"Brooke, J. (1996). SUS-A quick and dirty usability scale. Usability Evaluation in Industry, 189(194), 4\u20137.","journal-title":"Usability Evaluation in Industry,"},{"key":"e_1_3_2_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2015.03.043"},{"key":"e_1_3_2_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.apergo.2017.02.011"},{"key":"e_1_3_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979267"},{"key":"e_1_3_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/57167.57203"},{"key":"e_1_3_2_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2555805"},{"key":"e_1_3_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3148148"},{"key":"e_1_3_2_30_1","volume-title":"Mauroux. CrowdSearch 2012 Workshop at WWW 2012","author":"Difallah D. E.","year":"2012","unstructured":"Difallah, D. E., Demartini, G., Cudre-Mauroux, P. (2012). Mechanical cheat: Spamming schemes and adversarial techniques on crowdsourcing platforms. Mauroux. CrowdSearch 2012 Workshop at WWW 2012, Lyon, France."},{"key":"e_1_3_2_31_1","first-page":"147","article-title":"Enhancing the informatics evaluation toolkit with remote usability testing","volume":"2009","author":"Dixon B. E.","year":"2009","unstructured":"Dixon, B. E. (2009). Enhancing the informatics evaluation toolkit with remote usability testing. AMIA Annual Symposium Proceedings, 2009(November), 147\u2013151.","journal-title":"AMIA Annual Symposium Proceedings"},{"key":"e_1_3_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1924421.1924442"},{"key":"e_1_3_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753688"},{"key":"e_1_3_2_34_1","doi-asserted-by":"publisher","DOI":"10.1111\/1467-9817.00115"},{"key":"e_1_3_2_35_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.intcom.2010.04.004"},{"key":"e_1_3_2_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.paid.2012.11.030"},{"key":"e_1_3_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3119930"},{"key":"e_1_3_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702443"},{"key":"e_1_3_2_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-90238-4_3"},{"key":"e_1_3_2_40_1","unstructured":"Gamboa E. Libreros A. Hirth M. & Dubiner D. (2022). Human-AI collaboration for improving the identification of cars for autonomous driving. HIL-DC2022: ACM CIKM 2022 Workshop Human-In-The-Loop Data Curation October 22 2022 Atlanta Georgia."},{"key":"e_1_3_2_41_1","doi-asserted-by":"publisher","DOI":"10.1108\/AJIM-08-2020-0254"},{"issue":"1","key":"e_1_3_2_42_1","first-page":"575","article-title":"Affective crowdsourcing applied to usability testing","volume":"5","author":"Gomide V. H.","year":"2014","unstructured":"Gomide, V. H., Valle, P. A., Ferreira, J. O., Barbosa, J. R., Da Rocha, A. F., & Barbosa, T. (2014). Affective crowdsourcing applied to usability testing. International Journal of Computer Science and Information Technologies, 5(1), 575\u2013579.","journal-title":"International Journal of Computer Science and Information Technologies,"},{"key":"e_1_3_2_43_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2021.1898828"},{"key":"e_1_3_2_44_1","doi-asserted-by":"publisher","DOI":"10.1177\/0894439314533479"},{"key":"e_1_3_2_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2022.3207292"},{"key":"e_1_3_2_46_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.paid.2017.02.015"},{"key":"e_1_3_2_47_1","first-page":"191","article-title":"An experimental study on the effects of personalization, survey length statements, progress indicators, and survey sponsor logos in Web surveys","volume":"22","author":"Heerwegh D.","year":"2006","unstructured":"Heerwegh, D., & Loosveldt, G. (2006). An experimental study on the effects of personalization, survey length statements, progress indicators, and survey sponsor logos in Web surveys. Journal of Official Statistics, 22(2), 191\u2013210.","journal-title":"Journal of Official Statistics,"},{"key":"e_1_3_2_48_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2015.1065691"},{"key":"e_1_3_2_49_1","doi-asserted-by":"crossref","first-page":"157","DOI":"10.1037\/11428-008","volume-title":"Criminal profiling: Developing an effective science and practice","author":"Hicks S. J.","year":"2006","unstructured":"Hicks, S. J., & Sales, B. D. (2006). Offender characteristics: The constructs of motives, personality, and behavior. In S. J. Hicks & B. D. Sales (Eds.), Criminal profiling: Developing an effective science and practice (pp. 157\u2013170). American Psychological Association."},{"issue":"6","key":"e_1_3_2_50_1","first-page":"1","article-title":"The rise of crowdsourcing","volume":"14","author":"Howe J.","year":"2006","unstructured":"Howe, J. (2006). The rise of crowdsourcing. Wired Magazine, 14(6), 1\u20134.","journal-title":"Wired Magazine,"},{"key":"e_1_3_2_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00778-017-0484-3"},{"key":"e_1_3_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/503112.503114"},{"key":"e_1_3_2_53_1","first-page":"1127","volume-title":"Companion of the the Web Conference 2018","author":"Jim\u00e9nez R.","year":"2018","unstructured":"Jim\u00e9nez, R., Gallardo, L., & M\u00f6ller, S. (2018). Outliers detection vs. control questions to ensure reliable results in crowdsourcing: A speech quality assessment case study [Paper presentation]. Companion of the the Web Conference 2018, (pp. 1127\u20131130)."},{"key":"e_1_3_2_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2020.103351"},{"key":"e_1_3_2_55_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v3i1.13227"},{"key":"e_1_3_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063860"},{"key":"e_1_3_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2442576.2442591"},{"key":"e_1_3_2_58_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-8535.1993.tb00076.x"},{"key":"e_1_3_2_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_60_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2017.1336317"},{"key":"e_1_3_2_61_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2015.1064658"},{"key":"e_1_3_2_62_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2012.681221"},{"key":"e_1_3_2_63_1","doi-asserted-by":"publisher","DOI":"10.1086\/269128"},{"key":"e_1_3_2_64_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2020.1727262"},{"key":"e_1_3_2_65_1","volume-title":"Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010)","author":"Le J.","year":"2010","unstructured":"Le, J., Edmonds, A., Hester, V., Biewald, L. (2010). Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution. In Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010). ACM."},{"key":"e_1_3_2_66_1","doi-asserted-by":"publisher","DOI":"10.18148\/srm\/2019.v13i3.7403"},{"key":"e_1_3_2_67_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447319509526110"},{"key":"e_1_3_2_68_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2014.930311"},{"key":"e_1_3_2_69_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2017.1418805"},{"key":"e_1_3_2_70_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2018.1455307"},{"key":"e_1_3_2_71_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2018.1533152"},{"key":"e_1_3_2_72_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2015.1064654"},{"key":"e_1_3_2_73_1","doi-asserted-by":"publisher","DOI":"10.2196\/11066"},{"issue":"1","key":"e_1_3_2_74_1","first-page":"4","article-title":"The theory of Chinese reading speed","volume":"2","author":"Liao J.","year":"1996","unstructured":"Liao, J., & Zhang, W. (1996). The theory of Chinese reading speed. Chinese Journal of Ergonomics, 2(1), 4.","journal-title":"Chinese Journal of Ergonomics,"},{"key":"e_1_3_2_75_1","doi-asserted-by":"publisher","DOI":"10.1080\/07380560903095162"},{"key":"e_1_3_2_76_1","doi-asserted-by":"publisher","DOI":"10.1002\/meet.14504901100"},{"key":"e_1_3_2_77_1","doi-asserted-by":"publisher","DOI":"10.2196\/35319"},{"key":"e_1_3_2_78_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2011.540472"},{"key":"e_1_3_2_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSC.2017.7983605"},{"key":"e_1_3_2_80_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.09.015"},{"key":"e_1_3_2_81_1","volume-title":"Paper Presented at the Survey Research Methods","author":"Marquis L.","year":"2021","unstructured":"Marquis, L. (2021). Using response times to enhance the reliability of political knowledge items: An application to the 2015 Swiss post-election survey [Paper presentation]. Paper Presented at the Survey Research Methods."},{"key":"e_1_3_2_82_1","doi-asserted-by":"publisher","DOI":"10.1177\/154193120404800535"},{"key":"e_1_3_2_83_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0028085"},{"key":"e_1_3_2_84_1","doi-asserted-by":"publisher","DOI":"10.1016\/0191-8869(93)90047-7"},{"key":"e_1_3_2_85_1","doi-asserted-by":"publisher","DOI":"10.1089\/cpb.2007.0258"},{"key":"e_1_3_2_86_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2013.67"},{"key":"e_1_3_2_87_1","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2015-589"},{"key":"e_1_3_2_88_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.111122"},{"key":"e_1_3_2_89_1","doi-asserted-by":"publisher","DOI":"10.1109\/52.363161"},{"key":"e_1_3_2_90_1","volume-title":"Psychometric theory","author":"Nunnally J. C.","year":"1978","unstructured":"Nunnally, J. C. (1978). Psychometric theory (2nd ed.). Mcgraw Hill Book Company.","edition":"2"},{"key":"e_1_3_2_91_1","volume-title":"AAAI Conference on Human Computation","author":"Oleson D.","year":"2011","unstructured":"Oleson, D., Sorokin, A., Laughlin, G., Hester, V., Le, J., & Biewald, L. (2011). Programmatic gold: Targeted and scalable quality assurance in crowdsourcing [Paper presentation]. AAAI Conference on Human Computation. AAAI Press."},{"key":"e_1_3_2_92_1","doi-asserted-by":"publisher","DOI":"10.1002\/hfm.20316"},{"key":"e_1_3_2_93_1","first-page":"49","volume-title":"The role of constructs in psychological and educational measurement","author":"Paulhus D. L.","year":"2002","unstructured":"Paulhus, D. L. (2002). Social desirable responding: The evolution of a construct. In H. I. Braun, D. N. Jackson, & D. E. Wiley (Eds.), The role of constructs in psychological and educational measurement (pp. 49\u201369). Erlbaum."},{"key":"e_1_3_2_94_1","unstructured":"Rahman M. H. & Sj\u00f6str\u00f6m J. (2021). Respondent behavior logging: A design science research inquiry into web survey paradata [Paper presentation]. The Next Wave of Sociotechnical Design: 16th International Conference on Design Science Research in Information Systems and Technology DESRIST 2021 Kristiansand Norway August 4\u20136 2021 Proceedings 16."},{"key":"e_1_3_2_95_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.apergo.2018.08.011"},{"key":"e_1_3_2_96_1","volume-title":"Quantifying the user experience: Practical statistics for user research","author":"Lewis J.","year":"2014","unstructured":"Sauro., & Lewis, J. (2014). Quantifying the user experience: Practical statistics for user research. China Machine Press."},{"key":"e_1_3_2_97_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979266"},{"key":"e_1_3_2_98_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-32885-5_11"},{"key":"e_1_3_2_99_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-4951-5_44"},{"key":"e_1_3_2_100_1","doi-asserted-by":"publisher","DOI":"10.18148\/srm\/2015.v9i2.6128"},{"key":"e_1_3_2_101_1","doi-asserted-by":"publisher","DOI":"10.1080\/17517575.2018.1493539"},{"key":"e_1_3_2_102_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2010.05.013"},{"key":"e_1_3_2_103_1","doi-asserted-by":"publisher","DOI":"10.1111\/jlme.12200"},{"key":"e_1_3_2_104_1","doi-asserted-by":"publisher","DOI":"10.1145\/1029533.1029567"},{"key":"e_1_3_2_105_1","volume-title":"Handbook of usability testing: How to plan, design, and conduct effective tests","author":"VanArsdall E.","year":"2009","unstructured":"VanArsdall, E. (2009). Handbook of usability testing: How to plan, design, and conduct effective tests. JSTOR."},{"key":"e_1_3_2_106_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2021.1979291"},{"key":"e_1_3_2_107_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2021.1898826"},{"key":"e_1_3_2_108_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2019.1709339"},{"key":"e_1_3_2_109_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2019.1700644"},{"key":"e_1_3_2_110_1","doi-asserted-by":"publisher","DOI":"10.1145\/506443.506602"},{"key":"e_1_3_2_111_1","doi-asserted-by":"publisher","DOI":"10.2196\/humanfactors.4602"},{"key":"e_1_3_2_112_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICWS.2017.40"},{"key":"e_1_3_2_113_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2011.586320"},{"key":"e_1_3_2_114_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13160"}],"container-title":["International Journal of Human\u2013Computer Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.tandfonline.com\/doi\/pdf\/10.1080\/10447318.2023.2263694","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,14]],"date-time":"2024-11-14T06:16:37Z","timestamp":1731564997000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.tandfonline.com\/doi\/full\/10.1080\/10447318.2023.2263694"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,9]]},"references-count":113,"journal-issue":{"issue":"22","published-print":{"date-parts":[[2024,11,16]]}},"alternative-id":["10.1080\/10447318.2023.2263694"],"URL":"https:\/\/doi.org\/10.1080\/10447318.2023.2263694","relation":{},"ISSN":["1044-7318","1532-7590"],"issn-type":[{"type":"print","value":"1044-7318"},{"type":"electronic","value":"1532-7590"}],"subject":[],"published":{"date-parts":[[2023,10,9]]},"assertion":[{"value":"The publishing and review policy for this title is described in its Aims & Scope.","order":1,"name":"peerreview_statement","label":"Peer Review Statement"},{"value":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=hihc20","URL":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=hihc20","order":2,"name":"aims_and_scope_url","label":"Aim & Scope"},{"value":"2023-04-17","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-07-21","order":1,"name":"revised","label":"Revised","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-09-21","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-10-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}