{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:04:22Z","timestamp":1750309462709,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,30]],"date-time":"2024-09-30T00:00:00Z","timestamp":1727654400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,30]]},"DOI":"10.1145\/3688459.3688470","type":"proceedings-article","created":{"date-parts":[[2024,11,20]],"date-time":"2024-11-20T08:31:55Z","timestamp":1732091515000},"page":"309-321","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["A Metric to Assess the Reliability of Crowd-sourced SUS Scores: A Case Study on the PoPLar Authentication Tool"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7997-0063","authenticated-orcid":false,"given":"Yonas","family":"Leguesse","sequence":"first","affiliation":[{"name":"University of Malta, Msida, Malta,"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6483-9054","authenticated-orcid":false,"given":"Mark","family":"Vella","sequence":"additional","affiliation":[{"name":"University of Malta, Msida, Malta,"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2844-5728","authenticated-orcid":false,"given":"Christian","family":"Colombo","sequence":"additional","affiliation":[{"name":"University of Malta, Msida, Malta,"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6432-5328","authenticated-orcid":false,"given":"Julio","family":"Hernandez-Castro","sequence":"additional","affiliation":[{"name":"The Technical University of Madrid, Madrid, Spain,"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,11,20]]},"reference":[{"doi-asserted-by":"crossref","unstructured":"Aaron Bangor Philip Kortum and James Miller. 2008. An empirical evaluation of the system usability scale. International Journal of Human-Computer Interaction 24 6 (2008) 574\u2013594.","key":"e_1_3_3_2_2_2","DOI":"10.1080\/10447310802205776"},{"unstructured":"Aaron Bangor Philip Kortum and James Miller. 2009. Determining what individual SUS scores mean: Adding an adjective rating scale. Journal of usability studies 4 3 (2009) 114\u2013123.","key":"e_1_3_3_2_3_2"},{"unstructured":"John Brooke. 1986. System usability scale (SUS): a quick-and-dirty method of system evaluation user information. Reading UK: Digital equipment co ltd 43 (1986) 1\u20137.","key":"e_1_3_3_2_4_2"},{"unstructured":"John Brooke. 2013. SUS: a retrospective. Journal of usability studies 8 2 (2013) 29\u201340.","key":"e_1_3_3_2_5_2"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_6_2","DOI":"10.1007\/978-3-662-44811-3_22"},{"doi-asserted-by":"crossref","unstructured":"Attaullah Buriro Bruno Crispo and Mauro Conti. 2019. AnswerAuth: A bimodal behavioral biometric-based user authentication scheme for smartphones. Journal of information security and applications 44 (2019) 89\u2013103.","key":"e_1_3_3_2_7_2","DOI":"10.1016\/j.jisa.2018.11.008"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_8_2","DOI":"10.1109\/SPW.2016.20"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_9_2","DOI":"10.1145\/3176258.3176318"},{"doi-asserted-by":"crossref","unstructured":"Michael Chmielewski and Sarah\u00a0C Kucker. 2020. An MTurk crisis? Shifts in data quality and the impact on study results. Social Psychological and Personality Science 11 4 (2020) 464\u2013473.","key":"e_1_3_3_2_10_2","DOI":"10.1177\/1948550619875149"},{"doi-asserted-by":"crossref","unstructured":"Benjamin\u00a0D Douglas Patrick\u00a0J Ewell and Markus Brauer. 2023. Data quality in online human-subjects research: Comparisons between MTurk Prolific CloudResearch Qualtrics and SONA. Plos one 18 3 (2023) e0279720.","key":"e_1_3_3_2_11_2","DOI":"10.1371\/journal.pone.0279720"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_12_2","DOI":"10.1145\/3427228.3427252"},{"doi-asserted-by":"crossref","unstructured":"Kelly Grindrod Hassan Khan Urs Hengartner Stephanie Ong Alexander\u00a0G Logan Daniel Vogel Robert Gebotys and Jilan Yang. 2018. Evaluating authentication options for mobile health applications in younger and older adults. PloS one 13 1 (2018) e0189048.","key":"e_1_3_3_2_13_2","DOI":"10.1371\/journal.pone.0189048"},{"doi-asserted-by":"crossref","unstructured":"Nabeela Kausar Ikram\u00a0Ud Din Mudassar\u00a0Ali Khan Ahmad Almogren and Byung-Seo Kim. 2022. GRA-PIN: A Graphical and PIN-Based Hybrid Authentication Approach for Smart Devices. Sensors 22 4 (2022) 1349.","key":"e_1_3_3_2_14_2","DOI":"10.3390\/s22041349"},{"doi-asserted-by":"crossref","unstructured":"Ryan Kennedy Scott Clifford Tyler Burleigh Philip\u00a0D Waggoner Ryan Jewell and Nicholas\u00a0JG Winter. 2020. The shape of and solutions to the MTurk quality crisis. Political Science Research and Methods 8 4 (2020) 614\u2013629.","key":"e_1_3_3_2_15_2","DOI":"10.1017\/psrm.2020.6"},{"doi-asserted-by":"crossref","unstructured":"Oksana Kulyk Stephan Neumann Jurlind Budurushi and Melanie Volkamer. 2017. Nothing comes for free: How much usability can you sacrifice for security? IEEE Security & Privacy 15 3 (2017) 24\u201329.","key":"e_1_3_3_2_16_2","DOI":"10.1109\/MSP.2017.70"},{"doi-asserted-by":"crossref","unstructured":"Yonas Leguesse Christian Colombo Mark Vella and Julio Hernandez-Castro. 2021. PoPL: Proof-of-Presence and Locality or How to Secure Financial Transactions on Your Smartphone. IEEE Access 9 (2021) 168600\u2013168612.","key":"e_1_3_3_2_17_2","DOI":"10.1109\/ACCESS.2021.3137360"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_18_2","DOI":"10.1145\/2628363.2628372"},{"doi-asserted-by":"crossref","unstructured":"Markus Lennartsson Joakim K\u00e4vrestad and Marcus Nohlberg. 2021. Exploring the meaning of usable security\u2013a literature review. Information & Computer Security 29 4 (2021) 647\u2013663.","key":"e_1_3_3_2_19_2","DOI":"10.1108\/ICS-10-2020-0167"},{"doi-asserted-by":"crossref","unstructured":"James\u00a0R Lewis and Jeff Sauro. 2018. Measuring perceived usability: The SUS UMUX-LITE and AltUsability metric. International Journal of Human-Computer Interaction 34 6 (2018) 561\u2013567.","key":"e_1_3_3_2_20_2","DOI":"10.1080\/10447318.2017.1418805"},{"unstructured":"James\u00a0R Lewis and Jeff Sauro. 2019. Item Benchmarks for the System Usability Scale. Journal of Usability Studies 14 3 (2019) 149\u2013173.","key":"e_1_3_3_2_21_2"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_22_2","DOI":"10.1145\/3300061.3345434"},{"doi-asserted-by":"crossref","unstructured":"Di Liu Randolph\u00a0G Bias Matthew Lease and Rebecca Kuipers. 2012. Crowdsourcing for usability testing. Proceedings of the American Society for Information Science and Technology 49 1 (2012) 1\u201310.","key":"e_1_3_3_2_23_2","DOI":"10.1002\/meet.14504901100"},{"unstructured":"Sajaad\u00a0Ahmed Lone and AH Mir. 2022. Smartphone-based Biometric Authentication Scheme for Access Control Management in Client-server Environment. (2022).","key":"e_1_3_3_2_24_2"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_25_2","DOI":"10.1109\/SP40000.2020.00047"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_26_2","DOI":"10.1145\/3481357.3481515"},{"doi-asserted-by":"crossref","unstructured":"Christopher\u00a0D Mellinger and Thomas\u00a0A Hanson. 2020. Methodological considerations for survey research: Validity reliability and quantitative analysis. Linguistica Antverpiensia New Series\u2013Themes in Translation Studies 19 (2020).","key":"e_1_3_3_2_27_2","DOI":"10.52034\/lanstts.v19i0.549"},{"doi-asserted-by":"crossref","unstructured":"Toan Nguyen and Nasir Memon. 2018. Tap-based user authentication for smartwatches. Computers & Security 78 (2018) 174\u2013186.","key":"e_1_3_3_2_28_2","DOI":"10.1016\/j.cose.2018.07.001"},{"unstructured":"Wataru Oogami Hidehito Gomi Shuji Yamaguchi Shota Yamanaka and Tatsuru Higurashi. 2020. Observation study on usability challenges for fingerprint authentication using WebAuthn-enabled android smartphones. Age 20 (2020) 29.","key":"e_1_3_3_2_29_2"},{"key":"e_1_3_3_2_30_2","first-page":"57","volume-title":"Seventeenth Symposium on Usable Privacy and Security (SOUPS 2021)","author":"Owens Kentrell","year":"2021","unstructured":"Kentrell Owens, Olabode Anise, Amanda Krauss, and Blase Ur. 2021. User perceptions of the usability and security of smartphones as { FIDO2} roaming authenticators. In Seventeenth Symposium on Usable Privacy and Security (SOUPS 2021). 57\u201376."},{"key":"e_1_3_3_2_31_2","first-page":"357","volume-title":"Fifteenth Symposium on Usable Privacy and Security (SOUPS 2019)","author":"Reese Ken","year":"2019","unstructured":"Ken Reese, Trevor Smith, Jonathan Dutson, Jonathan Armknecht, Jacob Cameron, and Kent Seamons. 2019. A usability study of five { two-factor} authentication methods. In Fifteenth Symposium on Usable Privacy and Security (SOUPS 2019). 357\u2013370."},{"unstructured":"Steven\u00a0V Rouse. 2019. Reliability of MTurk data from masters and workers. Journal of Individual Differences (2019).","key":"e_1_3_3_2_32_2"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_33_2","DOI":"10.1145\/2736277.2741683"},{"key":"e_1_3_3_2_34_2","volume-title":"Twelfth Symposium on Usable Privacy and Security (SOUPS 2016)","author":"Ruoti Scott","year":"2016","unstructured":"Scott Ruoti and Kent Seamons. 2016. Standard metrics and scenarios for usable authentication. In Twelfth Symposium on Usable Privacy and Security (SOUPS 2016)."},{"unstructured":"Jeff Sauro and James\u00a0R Lewis. 2011. Measuring usability with the System Usability Scale (SUS). J Usability Studies 4 3 (2011) 194\u2013198.","key":"e_1_3_3_2_35_2"},{"doi-asserted-by":"publisher","key":"e_1_3_3_2_36_2","DOI":"10.1145\/2420950.2420976"},{"doi-asserted-by":"crossref","unstructured":"Bingbing Zhang and Sherice Gearhart. 2020. Collecting online survey data: A comparison of data quality among a commercial panel & MTurk. Survey Practice 13 1 (2020) 1\u201310.","key":"e_1_3_3_2_37_2","DOI":"10.29115\/SP-2020-0015"},{"doi-asserted-by":"crossref","unstructured":"Tiantian Zhu Zhengyang Qu Haitao Xu Jingsi Zhang Zhengyue Shao Yan Chen Sandeep Prabhakar and Jianfeng Yang. 2019. RiskCog: Unobtrusive real-time user authentication on mobile devices in the wild. IEEE Transactions on Mobile Computing 19 2 (2019) 466\u2013483.","key":"e_1_3_3_2_38_2","DOI":"10.1109\/TMC.2019.2892440"}],"event":{"acronym":"EuroUSEC 2024","name":"EuroUSEC 2024: The 2024 European Symposium on Usable Security","location":"Karlstad Sweden"},"container-title":["Proceedings of the 2024 European Symposium on Usable Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3688459.3688470","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3688459.3688470","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:10:29Z","timestamp":1750295429000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3688459.3688470"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,30]]},"references-count":37,"alternative-id":["10.1145\/3688459.3688470","10.1145\/3688459"],"URL":"https:\/\/doi.org\/10.1145\/3688459.3688470","relation":{},"subject":[],"published":{"date-parts":[[2024,9,30]]},"assertion":[{"value":"2024-11-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}