{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T07:54:05Z","timestamp":1770969245770,"version":"3.50.1"},"publisher-location":"Cham","reference-count":36,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031703959","type":"print"},{"value":"9783031703966","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-70396-6_15","type":"book-chapter","created":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T10:01:54Z","timestamp":1725184914000},"page":"256-272","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["What\u2019s Behind the\u00a0Screen? Unveiling UI Hierarchies in\u00a0Process-Related UI Logs"],"prefix":"10.1007","author":[{"given":"A.","family":"Mart\u00ednez-Rojas","sequence":"first","affiliation":[]},{"given":"A.","family":"Rodr\u00edguez-Ru\u00edz","sequence":"additional","affiliation":[]},{"given":"J. G.","family":"Enr\u00edquez","sequence":"additional","affiliation":[]},{"given":"A.","family":"Jim\u00e9nez-Ram\u00edrez","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,9,2]]},"reference":[{"key":"15_CR1","doi-asserted-by":"publisher","unstructured":"Mart\u00ednez-Rojas, A., Rodr\u00edguez-Ru\u00edz, A., Enr\u00edquez, J.G., Jim\u00e9nez-Ram\u00edrez, A.: A Taxonomy classifying UI components in desktop interfaces (2024). https:\/\/doi.org\/10.5281\/zenodo.10821377","DOI":"10.5281\/zenodo.10821377"},{"key":"15_CR2","doi-asserted-by":"publisher","unstructured":"Mart\u00ednez-Rojas, A., Rodr\u00edguez-Ru\u00edz, A., Enr\u00edquez, J.G., Jim\u00e9nez-Ram\u00edrez, A.: Annotated UI element dataset for desktop environments (2024). https:\/\/doi.org\/10.5281\/zenodo.10822752","DOI":"10.5281\/zenodo.10822752"},{"key":"15_CR3","doi-asserted-by":"crossref","unstructured":"Van\u00a0der Aalst, W.M., Bichler, M., Heinzl, A.: Robotic process automation (2018)","DOI":"10.1007\/s12599-018-0542-4"},{"key":"15_CR4","doi-asserted-by":"publisher","unstructured":"Abb, L., Rehse, J.R.: A reference data model for process-related user interaction logs. In: Di Ciccio, C., Dijkman, R., del R\u00edo Ortega, A., Rinderle-Ma, S. (eds.) Business Process Management. BPM 2022. LNCS, vol. 13420, pp. 57\u201374. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-16103-2_7","DOI":"10.1007\/978-3-031-16103-2_7"},{"key":"15_CR5","doi-asserted-by":"publisher","DOI":"10.1016\/j.compind.2022.103721","volume":"142","author":"S Agostinelli","year":"2022","unstructured":"Agostinelli, S., Lupia, M., Marrella, A., Mecella, M.: Reactive synthesis of software robots in RPA from user interface logs. Comput. Ind. 142, 103721 (2022)","journal-title":"Comput. Ind."},{"key":"15_CR6","doi-asserted-by":"crossref","unstructured":"A\u015firo\u011flu, B., et al.: A deep learning based object detection system for user interface code generation. In: 2022 International Congress on Human-Computer Interaction, Optimization and Robotic Applications (HORA), pp.\u00a01\u20135. IEEE (2022)","DOI":"10.1109\/HORA55278.2022.9799941"},{"key":"15_CR7","doi-asserted-by":"crossref","unstructured":"Beltramelli, T.: Pix2code: generating code from a graphical user interface screenshot. In: Proceedings of the ACM SIGCHI Symposium on Engineering Interactive Computing Systems, pp.\u00a01\u20136 (2018)","DOI":"10.1145\/3220134.3220135"},{"key":"15_CR8","unstructured":"Bochkovskiy, A., Wang, C.Y., Liao, H.Y.M.: Yolov4: optimal speed and accuracy of object detection. arXiv preprint arXiv:2004.10934 (2020)"},{"key":"15_CR9","unstructured":"Capgemini, C.: Robotic process automation-robots conquer business processes in back offices (2017)"},{"key":"15_CR10","doi-asserted-by":"crossref","unstructured":"Chen, J., et al.: Object detection for graphical user interface: old fashioned or deep learning or a combination? In: Proceedings of 28th ACM joint meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 1202\u20131214 (2020)","DOI":"10.1145\/3368089.3409691"},{"key":"15_CR11","doi-asserted-by":"crossref","unstructured":"Deka, B., et al.: Rico: a mobile app dataset for building data-driven design applications. In: Proceedings of the 30th Annual ACM Symposium on User Interface Software and Technology, pp. 845\u2013854 (2017)","DOI":"10.1145\/3126594.3126651"},{"key":"15_CR12","doi-asserted-by":"crossref","unstructured":"Dumas, M., La\u00a0Rosa, M., Leno, V., Polyvyanyy, A., Maggi, F.M.: Robotic process mining. Process Min. Handb. 468\u2013491 (2022)","DOI":"10.1007\/978-3-031-08848-3_16"},{"key":"15_CR13","doi-asserted-by":"crossref","unstructured":"Feiz, S., Wu, J., Zhang, X., Swearngin, A., Barik, T., Nichols, J.: Understanding screen relationships from screenshots of smartphone applications. In: 27th International Conference on Intelligent User Interfaces, pp. 447\u2013458 (2022)","DOI":"10.1145\/3490099.3511109"},{"key":"15_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"53","DOI":"10.1007\/978-3-030-86365-4_5","volume-title":"Artificial Neural Networks and Machine Learning \u2013 ICANN 2021","author":"Z Feng","year":"2021","unstructured":"Feng, Z., Fang, J., Cai, B., Zhang, Y.: GUIS2Code: a computer vision tool to\u00a0generate code automatically from graphical user interface sketches. In: Farka\u0161, I., Masulli, P., Otte, S., Wermter, S. (eds.) ICANN 2021. LNCS, vol. 12893, pp. 53\u201365. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-86365-4_5"},{"key":"15_CR15","doi-asserted-by":"crossref","unstructured":"Ji, W., Li, J., Bi, Q., Li, W., Cheng, L.: Segment anything is not always perfect: an investigation of SAM on different real-world applications. arXiv preprint arXiv:2304.05750 (2023)","DOI":"10.1007\/s11633-023-1385-0"},{"key":"15_CR16","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-3-030-85867-4_1","volume-title":"Business Process Management: Blockchain and Robotic Process Automation Forum","author":"A Jim\u00e9nez-Ram\u00edrez","year":"2021","unstructured":"Jim\u00e9nez-Ram\u00edrez, A.: Humans, processes and robots: a\u00a0journey to hyperautomation. In: Gonz\u00e1lez Enr\u00edquez, J., Debois, S., Fettke, P., Plebani, P., van de Weerd, I., Weber, I. (eds.) BPM 2021. LNBIP, vol. 428, pp. 3\u20136. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-85867-4_1"},{"key":"15_CR17","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"446","DOI":"10.1007\/978-3-030-21290-2_28","volume-title":"Advanced Information Systems Engineering","author":"A Jimenez-Ramirez","year":"2019","unstructured":"Jimenez-Ramirez, A., Reijers, H.A., Barba, I., Del Valle, C.: A method to improve the early stages of the robotic process automation lifecycle. In: Giorgini, P., Weber, B. (eds.) CAiSE 2019. LNCS, vol. 11483, pp. 446\u2013461. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-21290-2_28"},{"key":"15_CR18","unstructured":"Jocher, G., Chaurasia, A., Qiu, J.: Ultralytics yolov8 (2023). https:\/\/github.com\/ultralytics\/ultralytics"},{"issue":"1","key":"15_CR19","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1016\/j.infsof.2008.09.009","volume":"51","author":"B Kitchenham","year":"2009","unstructured":"Kitchenham, B., Brereton, O.P., Budgen, D., Turner, M., Bailey, J., Linkman, S.: Systematic literature reviews in software engineering-a systematic literature review. Inf. Softw. Technol. 51(1), 7\u201315 (2009)","journal-title":"Inf. Softw. Technol."},{"key":"15_CR20","unstructured":"Lacity, M., Willcocks, L.: What knowledge workers stand to gain from automation [online] Harvard business review (2015). https:\/\/hbr.org\/2015\/06\/what-knowledgeworkers-stand-to-gain-from-automation. Accessed 26 Jan 2018"},{"key":"15_CR21","unstructured":"Lee, A.: Generating webpages from screenshots (2018)"},{"key":"15_CR22","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2021.101916","volume":"107","author":"V Leno","year":"2022","unstructured":"Leno, V., Augusto, A., Dumas, M., La Rosa, M., Maggi, F.M., Polyvyanyy, A.: Discovering data transfer routines from user interaction logs. Inf. Syst. 107, 101916 (2022)","journal-title":"Inf. Syst."},{"key":"15_CR23","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1007\/s12599-020-00641-4","volume":"63","author":"V Leno","year":"2021","unstructured":"Leno, V., Polyvyanyy, A., Dumas, M., La Rosa, M., Maggi, F.M.: Robotic process mining: vision and challenges. Bus. Inf. Syst. Eng. 63, 301\u2013314 (2021)","journal-title":"Bus. Inf. Syst. Eng."},{"issue":"24","key":"15_CR24","doi-asserted-by":"publisher","first-page":"5665","DOI":"10.3390\/rs15245665","volume":"15","author":"Y Liu","year":"2023","unstructured":"Liu, Y., et al.: Mobilesam-track: lightweight one-shot tracking and segmentation of small objects on edge devices. Remote Sens. 15(24), 5665 (2023)","journal-title":"Remote Sens."},{"key":"15_CR25","doi-asserted-by":"crossref","unstructured":"Maire, M., Yu, S.X., Perona, P.: Hierarchical scene annotation (2013)","DOI":"10.5244\/C.27.84"},{"key":"15_CR26","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2023.102340","volume":"121","author":"A Mart\u00ednez-Rojas","year":"2024","unstructured":"Mart\u00ednez-Rojas, A., Jim\u00e9nez-Ram\u00edrez, A., Enr\u00edquez, J., Reijers, H.: A screenshot-based task mining framework for disclosing the drivers behind variable human actions. Inf. Syst. 121, 102340 (2024)","journal-title":"Inf. Syst."},{"key":"15_CR27","doi-asserted-by":"publisher","unstructured":"Mart\u00ednez-Rojas, A., Reijers, H.A., Jim\u00e9nez-Ram\u00edrez, A., Enr\u00edquez, J.G.: What are you gazing at? An approach to use eye-tracking for robotic process automation. In: K\u00f6pke, J., et al. (eds.) Business Process Management: Blockchain, Robotic Process Automation and Educators Forum. BPM 2023. LNBIP, vol. 491, pp. 120\u2013134. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-43433-4_8","DOI":"10.1007\/978-3-031-43433-4_8"},{"key":"15_CR28","doi-asserted-by":"publisher","unstructured":"Mart\u00ednez-Rojas, A., Jim\u00e9nez-Ram\u00edrez, A., Enr\u00edquez, J.G., Reijers, H.A.: Analyzing variable human actions for robotic process automation. In: Di Ciccio, C., Dijkman, R., del R\u00edo Ortega, A., Rinderle-Ma, S. (eds.) Business Process Management. BPM 2022. LNCS, vol. 13420, pp. 75\u201390. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-16103-2_8","DOI":"10.1007\/978-3-031-16103-2_8"},{"issue":"2","key":"15_CR29","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1109\/TSE.2018.2844788","volume":"46","author":"K Moran","year":"2018","unstructured":"Moran, K., Bernal-C\u00e1rdenas, C., Curcio, M., Bonett, R., Poshyvanyk, D.: Machine learning-based prototyping of graphical user interfaces for mobile apps. IEEE Trans. Softw. Eng. 46(2), 196\u2013221 (2018)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"15_CR30","unstructured":"Moritz, P., et\u00a0al.: Ray: a distributed framework for emerging $$\\{$$AI$$\\}$$ applications. In: OSDI 18, pp. 561\u2013577 (2018)"},{"key":"15_CR31","unstructured":"Ren, S., He, K., Girshick, R., Sun, J.: Faster R-CNN: towards real-time object detection with region proposal networks. Adv. Neural Inf. Process. Syst. 28 (2015)"},{"key":"15_CR32","doi-asserted-by":"crossref","unstructured":"Wu, J., Zhang, X., Nichols, J., Bigham, J.P.: Screen parsing: towards reverse engineering of UI models from screenshots. In: The 34th Annual ACM Symposium on User Interface Software and Technology, pp. 470\u2013483 (2021)","DOI":"10.1145\/3472749.3474763"},{"key":"15_CR33","doi-asserted-by":"crossref","unstructured":"Xie, M., Feng, S., Xing, Z., Chen, J., Chen, C.: UIED: a hybrid tool for GUI element detection. In: Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 1655\u20131659 (2020)","DOI":"10.1145\/3368089.3417940"},{"key":"15_CR34","doi-asserted-by":"publisher","unstructured":"Yao, X., Yap, M.H., Zhang, Y.: Towards a deep learning approach for automatic GUI layout generation. In: Bashir, A.K., Fortino, G., Khanna, A., Gupta, D. (eds.) Proceedings of International Conference on Computing and Communication Networks. LNNS, vol. 394, pp. 19\u201327. Springer, Singapore (2022). https:\/\/doi.org\/10.1007\/978-981-19-0604-6_2","DOI":"10.1007\/978-981-19-0604-6_2"},{"issue":"6","key":"15_CR35","doi-asserted-by":"publisher","first-page":"1245","DOI":"10.1137\/0218082","volume":"18","author":"K Zhang","year":"1989","unstructured":"Zhang, K., Shasha, D.: Simple fast algorithms for the editing distance between trees and related problems. SIAM J. Comput. 18(6), 1245\u20131262 (1989)","journal-title":"SIAM J. Comput."},{"key":"15_CR36","unstructured":"Zhao, X., et al.: Fast segment anything. arXiv preprint arXiv:2306.12156 (2023)"}],"container-title":["Lecture Notes in Computer Science","Business Process Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-70396-6_15","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T10:14:33Z","timestamp":1725185673000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-70396-6_15"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031703959","9783031703966"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-70396-6_15","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"2 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"BPM","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Business Process Management","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Krakow","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Poland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"bpm2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/bpm2024.agh.edu.pl\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}