{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T13:33:22Z","timestamp":1774359202565,"version":"3.50.1"},"reference-count":41,"publisher":"Elsevier BV","issue":"3","license":[{"start":{"date-parts":[[2022,9,1]],"date-time":"2022-09-01T00:00:00Z","timestamp":1661990400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2022,9,1]],"date-time":"2022-09-01T00:00:00Z","timestamp":1661990400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2022,5,30]],"date-time":"2022-05-30T00:00:00Z","timestamp":1653868800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["High-Confidence Computing"],"published-print":{"date-parts":[[2022,9]]},"DOI":"10.1016\/j.hcc.2022.100069","type":"journal-article","created":{"date-parts":[[2022,6,8]],"date-time":"2022-06-08T02:35:59Z","timestamp":1654655759000},"page":"100069","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":4,"title":["VORI: A framework for testing voice user interface interactability"],"prefix":"10.1016","volume":"2","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2275-0729","authenticated-orcid":false,"given":"Abrar S.","family":"Alrumayh","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chiu C.","family":"Tan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"key":"10.1016\/j.hcc.2022.100069_bib0001","unstructured":"Amazon, https:\/\/alexa.amazon.com\/."},{"key":"10.1016\/j.hcc.2022.100069_bib0002","unstructured":"Google, https:\/\/store.google.com\/product\/google_home."},{"key":"10.1016\/j.hcc.2022.100069_bib0003","unstructured":"L. accessed 24-Jan-2021, https:\/\/www.fool.com\/investing\/2020\/01\/24\/60-million-americans-now-own-smart-speakers.aspx."},{"key":"10.1016\/j.hcc.2022.100069_bib0004","unstructured":"L. accessed 24-Jan-2021., https:\/\/techcrunch.com\/2019\/01\/04\/more-than-100-million-alexa-devices-have-been-sold\/."},{"key":"10.1016\/j.hcc.2022.100069_bib0005","unstructured":"voicebot.ai, (https:\/\/voicebot.ai\/2019\/02\/15\/google-assistant-actions-total-4253-in-january-2019-up-2-5x-in-past-year-but-7-5-the-total-number-alexa-skills-in-u-s\/). Accessed: 2020-11-20."},{"key":"10.1016\/j.hcc.2022.100069_bib0006","series-title":"Proceedings of the 27th {USENIX} Security Symposium ({USENIX} Security 18)","first-page":"33","article-title":"Skill squatting attacks on amazon alexa","author":"Kumar","year":"2018"},{"key":"10.1016\/j.hcc.2022.100069_bib0007","doi-asserted-by":"crossref","unstructured":"J. Kowalski, A. Jaskulska, K. Skorupska, K. Abramczuk, C. Biele, W. Kope\u0107, K. Marasek, Older adults and voice interaction: a pilot study with Google home, in: Proceedings of the Extended Abstracts of the 2019 CHI Conference on Human Factors in Computing Systems, 2019, pp. 1\u20136.","DOI":"10.1145\/3290607.3312973"},{"key":"10.1016\/j.hcc.2022.100069_bib0008","doi-asserted-by":"crossref","unstructured":"A. Pyae, T.N. Joelsson, Investigating the usability and user experiences of voice user interface: a case of google home smart speaker, in: Proceedings of the 20th International Conference on Human-Computer Interaction with Mobile Devices and Services Adjunct, 2018, pp. 127\u2013131.","DOI":"10.1145\/3236112.3236130"},{"key":"10.1016\/j.hcc.2022.100069_bib0009","doi-asserted-by":"crossref","unstructured":"B.R. Cowan, P. Doyle, J. Edwards, D. Garaialde, A. Hayes-Brady, H.P. Branigan, J. Cabral, L. Clark, What\u2019s in an accent? the impact of accented synthetic speech on lexical choice in human-machine dialogue, in: Proceedings of the 1st International Conference on Conversational User Interfaces, 2019, pp. 1\u20138.","DOI":"10.1145\/3342775.3342786"},{"key":"10.1016\/j.hcc.2022.100069_bib0010","series-title":"Proceedings of the 10th International Conference on Computing, Communication and Networking Technologies (ICCCNT)","first-page":"1","article-title":"User experience with smart voice assistants: the accent perspective","author":"Pal","year":"2019"},{"key":"10.1016\/j.hcc.2022.100069_bib0011","series-title":"Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems","first-page":"1","article-title":"Patterns for how users overcome obstacles in voice user interfaces","author":"Myers","year":"2018"},{"key":"10.1016\/j.hcc.2022.100069_bib0012","series-title":"Proceedings of the 2017 CHI conference extended abstracts on human factors in computing systems","first-page":"2853","article-title":"\u201d alexa is my new bff\u201d social roles, user satisfaction, and personification of the amazon echo","author":"Purington","year":"2017"},{"key":"10.1016\/j.hcc.2022.100069_bib0013","series-title":"Proceedings of the 2nd Conference on Conversational User Interfaces","first-page":"1","article-title":"What can I say? Effects of discoverability in vuis on task performance and user experience","author":"Kirschthaler","year":"2020"},{"key":"10.1016\/j.hcc.2022.100069_bib0014","series-title":"Advances in Speech Recognition","first-page":"61","article-title":"\u01e3your word is my command\u01e5: Google search by voice: a case study","author":"Schalkwyk","year":"2010"},{"key":"10.1016\/j.hcc.2022.100069_bib0015","series-title":"Proceedings of the 18th International Conference on Human-Computer Interaction with Mobile Devices and Services","first-page":"72","article-title":"What can I say? Addressing user experience challenges of a mobile voice user interface for accessibility","author":"Corbett","year":"2016"},{"key":"10.1016\/j.hcc.2022.100069_bib0016","doi-asserted-by":"crossref","unstructured":"A. Furqan, C. Myers, J. Zhu, Learnability through adaptive discovery tools in voice user interfaces, in: Proceedings of the CHI Conference Extended Abstracts on Human Factors in Computing Systems, 2017, pp. 1617\u20131623.","DOI":"10.1145\/3027063.3053166"},{"key":"10.1016\/j.hcc.2022.100069_bib0017","doi-asserted-by":"crossref","unstructured":"C. Lentzsch, S.J. Shah, B. Andow, M. Degeling, A. Das, W. Enck, Hey Alexa, is this skill safe?: Taking a closer look at the alexa skill ecosystem, Proceedings of the Network and Distributed Systems Security (NDSS) Symposium2021 (2021).","DOI":"10.14722\/ndss.2021.23111"},{"key":"10.1016\/j.hcc.2022.100069_bib0018","series-title":"Proceedings of the IEEE Conference on Communications and Network Security (CNS)","first-page":"1","article-title":"Understanding and mitigating privacy leaks from third-party smart speaker apps","author":"Alrumayh","year":"2021"},{"key":"10.1016\/j.hcc.2022.100069_bib0019","doi-asserted-by":"crossref","unstructured":"S. Liao, C. Wilson, L. Cheng, H. Hu, H. Deng, Measuring the effectiveness of privacy policies for voice assistant applications, in: Proceedings of the Annual Computer Security Applications Conference, 2020, pp. 856\u2013869.","DOI":"10.1145\/3427228.3427250"},{"key":"10.1016\/j.hcc.2022.100069_bib0020","doi-asserted-by":"crossref","unstructured":"L. Cheng, C. Wilson, S. Liao, J. Young, D. Dong, H. Hu, Dangerous skills got certified: measuring the trustworthiness of skill certification in voice personal assistant platforms, in: Proceedings of the ACM SIGSAC Conference on Computer and Communications Security, 2020, pp. 1699\u20131716.","DOI":"10.1145\/3372297.3423339"},{"key":"10.1016\/j.hcc.2022.100069_bib0021","unstructured":"Z. Guo, Z. Lin, P. Li, K. Chen, Skillexplorer: understanding the behavior of skills in large scale, in: Proceedings of the 29th {USENIX} Security Symposium ({USENIX} Security 20), 2020, pp. 2649\u20132666."},{"issue":"4","key":"10.1016\/j.hcc.2022.100069_bib0022","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3432233","article-title":"Verhealth: vetting medical voice applications through policy enforcement","volume":"4","author":"Shezan","year":"2020","journal-title":"Proc. ACM Interact. Mob. Wearable Ubiquitous Technol."},{"key":"10.1016\/j.hcc.2022.100069_bib0023","doi-asserted-by":"crossref","unstructured":"A.C. Morris, V. Maier, P. Green, From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition, in: Proceedings of the 8th International Conference on Spoken Language Processing, 2004.","DOI":"10.21437\/Interspeech.2004-668"},{"issue":"1","key":"10.1016\/j.hcc.2022.100069_bib0024","doi-asserted-by":"crossref","first-page":"57","DOI":"10.1007\/s42486-020-00055-x","article-title":"Emerging mobile apps: Challenges and open problems","volume":"3","author":"Alrumayh","year":"2021","journal-title":"CCF Trans. Pervasive Comput. Interact."},{"key":"10.1016\/j.hcc.2022.100069_bib0025","unstructured":"Best practices for sample utterances and custom slot type values, https:\/\/developer.amazon.com\/en-US\/docs\/alexa\/custom-skills\/best-practices-for-sample-utterances-and-custom-slot-type-values.html. Accessed: 2021-06-8XS."},{"key":"10.1016\/j.hcc.2022.100069_bib0026","unstructured":"Alexa skills store, (https:\/\/www.amazon.com\/alexa-skills\/b) Accessed: 2020-08-20."},{"key":"10.1016\/j.hcc.2022.100069_bib0027","unstructured":"Natural language understanding, (https:\/\/towardsdatascience.com\/how-amazon-alexa-works-your-guide-to-natural-language-processing-ai-7506004709d3). Accessed: 2021-03-25."},{"key":"10.1016\/j.hcc.2022.100069_bib0028","unstructured":"Developing your first skill, (https:\/\/developer.amazon.com\/en-US\/docs\/alexa\/alexa-skills-kit-sdk-for-nodejs\/develop-your-first-skill.html), Accessed: 2021-03-01."},{"key":"10.1016\/j.hcc.2022.100069_bib0029","unstructured":"Text to speech (tts) library for python, (https:\/\/pypi.org\/project\/pyttsx3\/), Accessed: 2021-03-01."},{"key":"10.1016\/j.hcc.2022.100069_bib0030","unstructured":"Speech recognition library for python, (https:\/\/pypi.org\/project\/SpeechRecognition\/), Accessed: 2021-03-01."},{"key":"10.1016\/j.hcc.2022.100069_bib0031","unstructured":"Selenium, (https:\/\/selenium-python.readthedocs.io), Accessed: 2020-08-20."},{"issue":"6","key":"10.1016\/j.hcc.2022.100069_bib0033","first-page":"103-108","article-title":"Design and evaluation of a short version of the user experience questionnaire (UEQ-S)","volume":"4","author":"Schrepp","year":"2017","journal-title":"Int. J. Interact. Multimed. Artif. Intell."},{"issue":"2","key":"10.1016\/j.hcc.2022.100069_bib0034","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1023\/A:1022390615396","article-title":"Expanding the mos: Development and psychometric evaluation of the mos-r and mos-x","volume":"6","author":"Polkosky","year":"2003","journal-title":"Int. J. Speech Technol."},{"key":"10.1016\/j.hcc.2022.100069_bib0035","series-title":"Proceedings of the 6th International Symposium of Chinese CHI","first-page":"11","article-title":"Assessing the utility of the system usability scale for evaluating voice-based user interfaces","author":"Ghosh","year":"2018"},{"key":"10.1016\/j.hcc.2022.100069_bib0036","unstructured":"Tlx @ nasa ames - home, (https:\/\/humansystems.arc.nasa.gov\/groups\/TLX\/), Accessed: 2020-07-23."},{"key":"10.1016\/j.hcc.2022.100069_bib0037","unstructured":"K. Hone, Usability measurement for speech systems: Sassi revisited, in: Proceedings of the SIGCHI Conference Paper, Toronto, 2014."},{"key":"10.1016\/j.hcc.2022.100069_bib0038","doi-asserted-by":"crossref","unstructured":"K.S. Hone, R. Graham, Towards a tool for the subjective assessment of speech system interfaces (SASSI) (2000).","DOI":"10.1017\/S1351324900002497"},{"key":"10.1016\/j.hcc.2022.100069_bib0039","unstructured":"M.D. Polkosky, Toward a social-cognitive psychology of speech technology: affective responses to speech-based e-service (2005)."},{"key":"10.1016\/j.hcc.2022.100069_bib0040","series-title":"Mediated Interpersonal Communication","first-page":"48","article-title":"Machines as mediators: the challenge of technology for interpersonal communication theory and research","author":"Polkosky","year":"2008"},{"key":"10.1016\/j.hcc.2022.100069_bib0041","series-title":"Proceedings of the CHI Conference on Human Factors in Computing Systems","first-page":"1","article-title":"Firefox voice: an open and extensible voice assistant built upon the web","author":"Cambre","year":"2021"},{"key":"10.1016\/j.hcc.2022.100069_bib0042","series-title":"Proceedings of the 42nd ACM SIGPLAN International Conference on Programming Language Design and Implementation","first-page":"312","article-title":"Diy assistant: a multi-modal end-user programmable virtual assistant","author":"Fischer","year":"2021"}],"container-title":["High-Confidence Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2667295222000216?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2667295222000216?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,9,9]],"date-time":"2025-09-09T08:39:36Z","timestamp":1757407176000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2667295222000216"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9]]},"references-count":41,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2022,9]]}},"alternative-id":["S2667295222000216"],"URL":"https:\/\/doi.org\/10.1016\/j.hcc.2022.100069","relation":{},"ISSN":["2667-2952"],"issn-type":[{"value":"2667-2952","type":"print"}],"subject":[],"published":{"date-parts":[[2022,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"VORI: A framework for testing voice user interface interactability","name":"articletitle","label":"Article Title"},{"value":"High-Confidence Computing","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.hcc.2022.100069","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2022 The Author(s). Published by Elsevier B.V. on behalf of Shandong University.","name":"copyright","label":"Copyright"}],"article-number":"100069"}}