{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T17:16:14Z","timestamp":1777655774215,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":57,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613904.3642777","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:38:06Z","timestamp":1715416686000},"page":"1-16","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":49,"title":["AXNav: Replaying Accessibility Tests from Natural Language"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9950-1953","authenticated-orcid":false,"given":"Maryam","family":"Taeb","sequence":"first","affiliation":[{"name":"FAMU-FSU College of Engineering, Florida State University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0935-4745","authenticated-orcid":false,"given":"Amanda","family":"Swearngin","sequence":"additional","affiliation":[{"name":"Apple Inc, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8951-2878","authenticated-orcid":false,"given":"Eldon","family":"Schoop","sequence":"additional","affiliation":[{"name":"Apple Inc, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2377-9550","authenticated-orcid":false,"given":"Ruijia","family":"Cheng","sequence":"additional","affiliation":[{"name":"Apple Inc, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0022-6512","authenticated-orcid":false,"given":"Yue","family":"Jiang","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6880-8546","authenticated-orcid":false,"given":"Jeffrey","family":"Nichols","sequence":"additional","affiliation":[{"name":"Apple Inc, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_3_1_1","unstructured":"Accessibility on iOS 2023. Accessibility on iOS. https:\/\/developer.apple.com\/accessibility\/"},{"key":"e_1_3_3_3_2_1","unstructured":"Accessibility Programming Guide 2022. Accessibility Programming Guide for OS X: Testing for Accessibility on OS X. https:\/\/developer.apple.com\/library\/archive\/documentation\/Accessibility\/Conceptual\/AccessibilityMacOSX\/OSXAXTestingApps.html"},{"key":"e_1_3_3_3_3_1","unstructured":"Accessibility Scanner 2023. Accessibility Scanner. https:\/\/play.google.com\/store\/apps\/details?id=com.google.android.apps.accessibility.auditor&hl=en_US"},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380392"},{"key":"e_1_3_3_3_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_3_3_6_1","unstructured":"Apple. 2020. Recognizing Text in Images. https:\/\/developer.apple.com\/documentation\/vision\/recognizing_text_in_images\/"},{"key":"e_1_3_3_3_7_1","volume-title":"Proceedings of the First Workshop on Bridging Human\u2013Computer Interaction and Natural Language Processing, Su\u00a0Lin Blodgett, Michael Madaio, Brendan O\u2019Connor","author":"Bhat Advait","year":"2021","unstructured":"Advait Bhat, Saaket Agashe, and Anirudha Joshi. 2021. How do people interact with biased text prediction models while writing?. In Proceedings of the First Workshop on Bridging Human\u2013Computer Interaction and Natural Language Processing, Su\u00a0Lin Blodgett, Michael Madaio, Brendan O\u2019Connor, Hanna Wallach, and Qian Yang (Eds.). Association for Computational Linguistics, Online, 116\u2013121. https:\/\/aclanthology.org\/2021.hcinlp-1.18"},{"key":"e_1_3_3_3_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3503508"},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1878803.1878812"},{"key":"e_1_3_3_3_10_1","unstructured":"Build accessible apps 2023. Build accessible apps. https:\/\/developer.android.com\/guide\/topics\/ui\/accessibility\/"},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"crossref","unstructured":"Andrea Burns Deniz Arsan Sanjna Agrawal Ranjitha Kumar Kate Saenko and Bryan\u00a0A. Plummer. 2022. A Dataset for Interactive Vision-Language Navigation with Unknown Command Feasibility. arxiv:2202.02312\u00a0[cs.CL]","DOI":"10.1007\/978-3-031-20074-8_18"},{"key":"e_1_3_3_3_12_1","unstructured":"Button Shapes 2023. Accessibility (Button Shapes). https:\/\/developer.apple.com\/design\/human-interface-guidelines\/accessibility"},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.1986.4767851"},{"key":"e_1_3_3_3_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502073"},{"key":"e_1_3_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3108162"},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610219"},{"key":"e_1_3_3_3_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-019-09701-0"},{"key":"e_1_3_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753554"},{"key":"e_1_3_3_3_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/361237.361242"},{"key":"e_1_3_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2018.00021"},{"key":"e_1_3_3_3_21_1","unstructured":"Espresso 2023. Espresso. https:\/\/developer.android.com\/training\/testing\/espresso"},{"key":"e_1_3_3_3_22_1","unstructured":"Sidong Feng and Chunyang Chen. 2023. Prompting Is All You Need: Automated Android Bug Replay with Large Language Models. arxiv:2306.01987\u00a0[cs.SE]"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502143"},{"key":"e_1_3_3_3_24_1","unstructured":"Get started on Android with Talkback 2023. Get started on Android with Talkback. https:\/\/support.google.com\/accessibility\/android\/answer\/6283677?hl=ens"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.4135\/9781483384436"},{"key":"e_1_3_3_3_26_1","volume-title":"Mustafa Safdari, Yutaka Matsuo, Douglas Eck, and Aleksandra Faust.","author":"Gur Izzeddin","year":"2023","unstructured":"Izzeddin Gur, Hiroki Furuta, Austin Huang, Mustafa Safdari, Yutaka Matsuo, Douglas Eck, and Aleksandra Faust. 2023. A Real-World WebAgent with Planning, Long Context Understanding, and Program Synthesis. arxiv:2307.12856\u00a0[cs.LG]"},{"key":"e_1_3_3_3_27_1","unstructured":"Improve your code 2023. Improve your code with lint checks. https:\/\/developer.android.com\/studio\/write\/lint?hl=en"},{"key":"e_1_3_3_3_28_1","unstructured":"Yue Jiang Eldon Schoop Amanda Swearngin and Jeffrey Nichols. 2023. ILuvUI: Instruction-tuned LangUage-Vision modeling of UIs from Machine Conversations. arxiv:2310.04869\u00a0[cs.HC]"},{"key":"e_1_3_3_3_29_1","unstructured":"Saurav Kadavath Tom Conerly Amanda Askell Tom Henighan Dawn Drain Ethan Perez Nicholas Schiefer Zac Hatfield-Dodds Nova DasSarma Eli Tran-Johnson Scott Johnston Sheer El-Showk Andy Jones Nelson Elhage Tristan Hume Anna Chen Yuntao Bai Sam Bowman Stanislav Fort Deep Ganguli Danny Hernandez Josh Jacobson Jackson Kernion Shauna Kravec Liane Lovitt Kamal Ndousse Catherine Olsson Sam Ringer Dario Amodei Tom Brown Jack Clark Nicholas Joseph Ben Mann Sam McCandlish Chris Olah and Jared Kaplan. 2022. Language Models (Mostly) Know What They Know. arxiv:2207.05221\u00a0[cs.CL]"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2015.7102609"},{"key":"e_1_3_3_3_31_1","volume-title":"Advances in Neural Information Processing Systems, S.\u00a0Koyejo, S.\u00a0Mohamed, A.\u00a0Agarwal, D.\u00a0Belgrave, K.\u00a0Cho, and A.\u00a0Oh (Eds.). Vol.\u00a035. Curran Associates","author":"Kojima Takeshi","year":"2022","unstructured":"Takeshi Kojima, Shixiang\u00a0(Shane) Gu, Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022. Large Language Models are Zero-Shot Reasoners. In Advances in Neural Information Processing Systems, S.\u00a0Koyejo, S.\u00a0Mohamed, A.\u00a0Agarwal, D.\u00a0Belgrave, K.\u00a0Cho, and A.\u00a0Oh (Eds.). Vol.\u00a035. Curran Associates, Inc., Red Hook, NY, USA, 22199\u201322213. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2022\/file\/8bb0d291acd4acf06ef112099c16f326-Paper-Conference.pdf"},{"key":"e_1_3_3_3_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2017.22"},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_3_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416623"},{"key":"e_1_3_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2017.47"},{"key":"e_1_3_3_3_36_1","unstructured":"Zhe Liu Chunyang Chen Junjie Wang Mengzhuo Chen Boyu Wu Xing Che Dandan Wang and Qing Wang. 2023. Chatting with GPT-3 for Zero-Shot Human-Like Mobile Automated GUI Testing. arxiv:2305.09434\u00a0[cs.SE]"},{"key":"e_1_3_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1054972.1054979"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.1979.4310076"},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2020.3007664"},{"key":"e_1_3_3_3_41_1","unstructured":"Christopher Rawles Alice Li Daniel Rodriguez Oriana Riva and Timothy Lillicrap. 2023. Android in the Wild: A Large-Scale Dataset for Android Device Control. arxiv:2307.10088\u00a0[cs.LG]"},{"key":"e_1_3_3_3_42_1","unstructured":"Roboelectric 2021. Roboelectric. http:\/\/robolectric.org\/"},{"key":"e_1_3_3_3_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132525.3132547"},{"key":"e_1_3_3_3_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445455"},{"key":"e_1_3_3_3_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580679"},{"key":"e_1_3_3_3_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556905"},{"key":"e_1_3_3_3_47_1","volume-title":"From Pixels to UI Actions: Learning to Follow Instructions via Graphical User Interfaces. Advances in Neural Information Processing Systems 36","author":"Shaw Peter","year":"2024","unstructured":"Peter Shaw, Mandar Joshi, James Cohan, Jonathan Berant, Panupong Pasupat, Hexiang Hu, Urvashi Khandelwal, Kenton Lee, and Kristina\u00a0N Toutanova. 2024. From Pixels to UI Actions: Learning to Follow Instructions via Graphical User Interfaces. Advances in Neural Information Processing Systems 36 (2024). https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/hash\/6c52a8a4fadc9129c6e1d1745f2dfd0f-Abstract-Conference.html"},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3218585.3218673"},{"key":"e_1_3_3_3_49_1","volume-title":"UGIF: UI Grounded Instruction Following. arxiv:2211.07615\u00a0[cs.CL]","author":"Venkatesh Sagar\u00a0Gubbi","year":"2023","unstructured":"Sagar\u00a0Gubbi Venkatesh, Partha Talukdar, and Srini Narayanan. 2023. UGIF: UI Grounded Instruction Following. arxiv:2211.07615\u00a0[cs.CL]"},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2461121.2461124"},{"key":"e_1_3_3_3_51_1","unstructured":"VoiceOver 2023. VoiceOver. https:\/\/support.apple.com\/guide\/iphone\/learn-voiceover-gestures-iph3e2e2281\/ios"},{"key":"e_1_3_3_3_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580895"},{"key":"e_1_3_3_3_53_1","unstructured":"WCAG 2 Overview 2023. WCAG 2 Overview. https:\/\/www.w3.org\/WAI\/standards-guidelines\/wcag\/"},{"key":"e_1_3_3_3_54_1","volume-title":"Shiqi Jiang, Yunhao Liu, Yaqin Zhang, and Yunxin Liu.","author":"Wen Hao","year":"2023","unstructured":"Hao Wen, Yuanchun Li, Guohong Liu, Shanhui Zhao, Tao Yu, Toby Jia-Jun Li, Shiqi Jiang, Yunhao Liu, Yaqin Zhang, and Yunxin Liu. 2023. Empowering LLM to use Smartphone for Intelligent Task Automation. arxiv:2308.15272\u00a0[cs.AI]"},{"key":"e_1_3_3_3_55_1","unstructured":"XCTest 2023. XCTest. https:\/\/developer.apple.com\/documentation\/xctest"},{"key":"e_1_3_3_3_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3300176"},{"key":"e_1_3_3_3_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445186"},{"key":"e_1_3_3_3_58_1","unstructured":"Zhizheng Zhang Xiaoyi Zhang Wenxuan Xie and Yan Lu. 2023. Responsible Task Automation: Empowering Large Language Models as Responsible Task Automators. arxiv:2306.01242\u00a0[cs.AI]"}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Proceedings of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642777","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613904.3642777","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:29Z","timestamp":1750290269000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613904.3642777"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":57,"alternative-id":["10.1145\/3613904.3642777","10.1145\/3613904"],"URL":"https:\/\/doi.org\/10.1145\/3613904.3642777","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}