{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,27]],"date-time":"2026-05-27T12:54:59Z","timestamp":1779886499310,"version":"3.53.1"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,10,10]],"date-time":"2021-10-10T00:00:00Z","timestamp":1633824000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,10,10]]},"DOI":"10.1145\/3472749.3474763","type":"proceedings-article","created":{"date-parts":[[2021,10,13]],"date-time":"2021-10-13T01:11:21Z","timestamp":1634087481000},"page":"470-483","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":61,"title":["Screen Parsing: Towards Reverse Engineering of UI Models from Screenshots"],"prefix":"10.1145","author":[{"given":"Jason","family":"Wu","sequence":"first","affiliation":[{"name":"Human-Computer Interaction Institute Carnegie Mellon University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoyi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Apple, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeff","family":"Nichols","sequence":"additional","affiliation":[{"name":"Apple, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffrey P","family":"Bigham","sequence":"additional","affiliation":[{"name":"Apple, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,10,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Airbnb. 2017. Sketching Interfaces. https:\/\/airbnb.design\/sketching-interfaces\/  Airbnb. 2017. Sketching Interfaces. https:\/\/airbnb.design\/sketching-interfaces\/"},{"key":"e_1_3_2_1_2_1","unstructured":"AutoIt. 2021. Function PixelSearch. https:\/\/www.autoitscript.com\/autoit3\/docs\/functions\/PixelSearch.htm  AutoIt. 2021. Function PixelSearch. https:\/\/www.autoitscript.com\/autoit3\/docs\/functions\/PixelSearch.htm"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Miguel Ballesteros Yoav Goldberg Chris Dyer and Noah\u00a0A Smith. 2016. Training with exploration improves a greedy stack-LSTM parser. arXiv preprint arXiv:1603.03793(2016).  Miguel Ballesteros Yoav Goldberg Chris Dyer and Noah\u00a0A Smith. 2016. Training with exploration improves a greedy stack-LSTM parser. arXiv preprint arXiv:1603.03793(2016).","DOI":"10.18653\/v1\/D16-1211"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3220134.3220135"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047228"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180240"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380327"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3126594.3126651"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753554"},{"key":"e_1_3_2_1_10_1","unstructured":"WA Falcon and .al. 2019. PyTorch Lightning. GitHub. Note: https:\/\/github.com\/PyTorchLightning\/pytorch-lightning 3(2019).  WA Falcon and .al. 2019. PyTorch Lightning. GitHub. Note: https:\/\/github.com\/PyTorchLightning\/pytorch-lightning 3(2019)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2016.06.014"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/964442.964461"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of COLING","author":"Goldberg Yoav","year":"2012","unstructured":"Yoav Goldberg and Joakim Nivre . 2012 . A dynamic oracle for arc-eager dependency parsing . In Proceedings of COLING 2012. 959\u2013976. Yoav Goldberg and Joakim Nivre. 2012. A dynamic oracle for arc-eager dependency parsing. In Proceedings of COLING 2012. 959\u2013976."},{"key":"e_1_3_2_1_14_1","unstructured":"W3C\u00a0Working Group. 2014. Introduction to Model-Based User Interfaces. https:\/\/www.w3.org\/TR\/mbui-intro\/  W3C\u00a0Working Group. 2014. Introduction to Model-Based User Interfaces. https:\/\/www.w3.org\/TR\/mbui-intro\/"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00306"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_17_1","volume-title":"Long short-term memory. Neural computation 9, 8","author":"Hochreiter Sepp","year":"1997","unstructured":"Sepp Hochreiter and J\u00fcrgen Schmidhuber . 1997. Long short-term memory. Neural computation 9, 8 ( 1997 ), 1735\u20131780. Sepp Hochreiter and J\u00fcrgen Schmidhuber. 1997. Long short-term memory. Neural computation 9, 8 (1997), 1735\u20131780."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-1162"},{"key":"e_1_3_2_1_19_1","volume-title":"Speech & language processing","author":"Jurafsky Dan","unstructured":"Dan Jurafsky and James\u00a0 H. Martin . 2020. Speech & language processing 3 rd ed. draft. Dan Jurafsky and James\u00a0H. Martin. 2020. Speech & language processing 3rd ed. draft.","edition":"3"},{"key":"e_1_3_2_1_20_1","unstructured":"Boris Knyazev Harm de Vries C\u0103t\u0103lina Cangea Graham\u00a0W Taylor Aaron Courville and Eugene Belilovsky. 2020. Graph density-aware losses for novel compositions in scene graph generation. arXiv preprint arXiv:2005.08230(2020).  Boris Knyazev Harm de Vries C\u0103t\u0103lina Cangea Graham\u00a0W Taylor Aaron Courville and Eugene Belilovsky. 2020. Graph density-aware losses for novel compositions in scene graph generation. arXiv preprint arXiv:2005.08230(2020)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"e_1_3_2_1_22_1","volume-title":"The Hungarian method for the assignment problem. Naval research logistics quarterly 2, 1-2","author":"Kuhn W","year":"1955","unstructured":"Harold\u00a0 W Kuhn . 1955. The Hungarian method for the assignment problem. Naval research logistics quarterly 2, 1-2 ( 1955 ), 83\u201397. Harold\u00a0W Kuhn. 1955. The Hungarian method for the assignment problem. Naval research logistics quarterly 2, 1-2 (1955), 83\u201397."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/257089.257396"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 2017 CHI conference on human factors in computing systems. 6038\u20136049","author":"Jia-Jun Li Toby","year":"2017","unstructured":"Toby Jia-Jun Li , Amos Azaria , and Brad\u00a0 A Myers . 2017 . SUGILITE: creating multimodal smartphone automation by demonstration . In Proceedings of the 2017 CHI conference on human factors in computing systems. 6038\u20136049 . Toby Jia-Jun Li, Amos Azaria, and Brad\u00a0A Myers. 2017. SUGILITE: creating multimodal smartphone automation by demonstration. In Proceedings of the 2017 CHI conference on human factors in computing systems. 6038\u20136049."},{"key":"e_1_3_2_1_25_1","unstructured":"Xuezhe Ma Zecong Hu Jingzhou Liu Nanyun Peng Graham Neubig and Eduard Hovy. 2018. Stack-pointer networks for dependency parsing. arXiv preprint arXiv:1805.01087(2018).  Xuezhe Ma Zecong Hu Jingzhou Liu Nanyun Peng Graham Neubig and Eduard Hovy. 2018. Stack-pointer networks for dependency parsing. arXiv preprint arXiv:1805.01087(2018)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_12"},{"key":"e_1_3_2_1_27_1","volume-title":"Umap: Uniform manifold approximation and projection for dimension reduction. arXiv preprint arXiv:1802.03426(2018).","author":"McInnes Leland","year":"2018","unstructured":"Leland McInnes , John Healy , and James Melville . 2018 . Umap: Uniform manifold approximation and projection for dimension reduction. arXiv preprint arXiv:1802.03426(2018). Leland McInnes, John Healy, and James Melville. 2018. Umap: Uniform manifold approximation and projection for dimension reduction. arXiv preprint arXiv:1802.03426(2018)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDAR.2005.182"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2015.32"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/571985.572008"},{"key":"e_1_3_2_1_31_1","unstructured":"Adam Paszke Sam Gross Soumith Chintala Gregory Chanan Edward Yang Zachary DeVito Zeming Lin Alban Desmaison Luca Antiga and Adam Lerer. 2017. Automatic differentiation in pytorch. (2017).  Adam Paszke Sam Gross Soumith Chintala Gregory Chanan Edward Yang Zachary DeVito Zeming Lin Alban Desmaison Luca Antiga and Adam Lerer. 2017. Automatic differentiation in pytorch. (2017)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/52.595902"},{"key":"e_1_3_2_1_33_1","unstructured":"Shaoqing Ren Kaiming He Ross Girshick and Jian Sun. 2015. Faster r-cnn: Towards real-time object detection with region proposal networks. arXiv preprint arXiv:1506.01497(2015).  Shaoqing Ren Kaiming He Ross Girshick and Jian Sun. 2015. Faster r-cnn: Towards real-time object detection with region proposal networks. arXiv preprint arXiv:1506.01497(2015)."},{"key":"e_1_3_2_1_34_1","unstructured":"Richard\u00a0S. Schwerdtfeger. 1991. Making the GUI Talk. ftp:\/\/service.boulder.ibm.com\/sns\/sr-os2\/sr2doc\/guitalk.txt  Richard\u00a0S. Schwerdtfeger. 1991. Making the GUI Talk. ftp:\/\/service.boulder.ibm.com\/sns\/sr-os2\/sr2doc\/guitalk.txt"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2005.140"},{"key":"e_1_3_2_1_36_1","volume-title":"Andrew\u00a0Y Ng, and Christopher\u00a0D Manning.","author":"Socher Richard","year":"2011","unstructured":"Richard Socher , Cliff Chiung-Yu Lin , Andrew\u00a0Y Ng, and Christopher\u00a0D Manning. 2011 . Parsing natural scenes and natural language with recursive neural networks. In ICML. Richard Socher, Cliff Chiung-Yu Lin, Andrew\u00a0Y Ng, and Christopher\u00a0D Manning. 2011. Parsing natural scenes and natural language with recursive neural networks. In ICML."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-005-6642-x"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_41"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1622176.1622213"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00611"},{"key":"e_1_3_2_1_41_1","volume-title":"Screen Recognition: Creating Accessibility Metadata for Mobile Applications from Pixels. arXiv preprint arXiv:2101.04893(2021).","author":"Zhang Xiaoyi","year":"2021","unstructured":"Xiaoyi Zhang , Lilian de Greef , Amanda Swearngin , Samuel White , Kyle Murray , Lisa Yu , Qi Shan , Jeffrey Nichols , Jason Wu , Chris Fleizach , 2021 . Screen Recognition: Creating Accessibility Metadata for Mobile Applications from Pixels. arXiv preprint arXiv:2101.04893(2021). Xiaoyi Zhang, Lilian de Greef, Amanda Swearngin, Samuel White, Kyle Murray, Lisa Yu, Qi Shan, Jeffrey Nichols, Jason Wu, Chris Fleizach, 2021. Screen Recognition: Creating Accessibility Metadata for Mobile Applications from Pixels. arXiv preprint arXiv:2101.04893(2021)."},{"key":"e_1_3_2_1_42_1","volume-title":"A stochastic grammar of images","author":"Zhu Song-Chun","unstructured":"Song-Chun Zhu and David Mumford . 2007. A stochastic grammar of images . Now Publishers Inc . Song-Chun Zhu and David Mumford. 2007. A stochastic grammar of images. Now Publishers Inc."}],"event":{"name":"UIST '21: The 34th Annual ACM Symposium on User Interface Software and Technology","location":"Virtual Event USA","acronym":"UIST '21","sponsor":["SIGGRAPH ACM Special Interest Group on Computer Graphics and Interactive Techniques","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["The 34th Annual ACM Symposium on User Interface Software and Technology"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3472749.3474763","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3472749.3474763","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:09Z","timestamp":1750191429000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3472749.3474763"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10,10]]},"references-count":42,"alternative-id":["10.1145\/3472749.3474763","10.1145\/3472749"],"URL":"https:\/\/doi.org\/10.1145\/3472749.3474763","relation":{},"subject":[],"published":{"date-parts":[[2021,10,10]]},"assertion":[{"value":"2021-10-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}