{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,2]],"date-time":"2026-08-02T05:45:58Z","timestamp":1785649558843,"version":"3.56.0"},"publisher-location":"Cham","reference-count":46,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032314031","type":"print"},{"value":"9783032314048","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,8,3]],"date-time":"2026-08-03T00:00:00Z","timestamp":1785715200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,8,3]],"date-time":"2026-08-03T00:00:00Z","timestamp":1785715200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-3-032-31404-8_10","type":"book-chapter","created":{"date-parts":[[2026,8,2]],"date-time":"2026-08-02T04:56:23Z","timestamp":1785646583000},"page":"139-154","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Leveraging Gaze and\u00a0Set-of-Mark in\u00a0VLLMs for\u00a0Human-Object Interaction Anticipation from\u00a0Egocentric Videos"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-3815-7333","authenticated-orcid":false,"given":"Daniele","family":"Materia","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6368-1910","authenticated-orcid":false,"given":"Francesco","family":"Ragusa","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6034-0432","authenticated-orcid":false,"given":"Giovanni Maria","family":"Farinella","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,8,3]]},"reference":[{"key":"10_CR1","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1007\/s11263-021-01531-2","volume":"130","author":"D Damen","year":"2022","unstructured":"Damen, D., et al.: Rescaling egocentric vision: collection, pipeline and challenges for epic-kitchens-100. Inter. J. Comput. Vis. (IJCV) 130, 33\u201355 (2022)","journal-title":"Inter. J. Comput. Vis. (IJCV)"},{"key":"10_CR2","doi-asserted-by":"crossref","unstructured":"Damen, D., Doughty, H., Farinella, G.M.: Scaling egocentric vision: the epic-kitchens dataset. In: Proceedings of the European Conference on Computer Vision (ECCV), pp. 720\u2013736 (2018)","DOI":"10.1007\/978-3-030-01225-0_44"},{"key":"10_CR3","unstructured":"Damen, D., Leelasawassuk, T.: You-do, i-learn: unsupervised multi-user egocentric approach towards video-based guidance arXiv:1510.04862 (2015)"},{"key":"10_CR4","doi-asserted-by":"publisher","first-page":"401","DOI":"10.1016\/j.jvcir.2017.10.004","volume":"49","author":"A Furnari","year":"2017","unstructured":"Furnari, A., Battiato, S., Grauman, K., Farinella, G.M.: Next-active-object prediction from egocentric videos. J. Vis. Commun. Image Represent. 49, 401\u2013411 (2017)","journal-title":"J. Vis. Commun. Image Represent."},{"issue":"11","key":"10_CR5","doi-asserted-by":"publisher","first-page":"4021","DOI":"10.1109\/TPAMI.2020.2992889","volume":"43","author":"A Furnari","year":"2021","unstructured":"Furnari, A., Farinella, G.M.: Rolling-unrolling lstms for action anticipation from first-person video. IEEE Trans. Pattern Anal. Mach. Intell. 43(11), 4021\u20134036 (2021)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"10_CR6","unstructured":"Google: Gemini 2.0 Flash (2024). https:\/\/docs.cloud.google.com\/vertex-ai\/generative-ai\/docs\/models\/gemini\/2-0-flash, Accessed: 24 Apr 2026"},{"key":"10_CR7","unstructured":"Google: Google AI API (2025). https:\/\/ai.google.dev\/api, Accessed 20 Jan 2026"},{"key":"10_CR8","unstructured":"Google: Retired gemini models (2025). https:\/\/cloud.google.com\/vertex-ai\/generative-ai\/docs\/learn\/model-versions#retired-models, Accessed 26 Apr 2026"},{"key":"10_CR9","unstructured":"Grauman, K., Westbury, A.: Ego4d: around the world in 3,000 hours of egocentric video. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 18995\u201319012 (2022)"},{"issue":"12","key":"10_CR10","doi-asserted-by":"publisher","first-page":"8356","DOI":"10.1007\/s11263-025-02557-6","volume":"133","author":"K Grauman","year":"2025","unstructured":"Grauman, K., et al.: Ego-exo4d: understanding skilled human activity from first- and third-person perspectives. Int. J. Comput. Vision 133(12), 8356\u20138435 (2025)","journal-title":"Int. J. Comput. Vision"},{"key":"10_CR11","doi-asserted-by":"crossref","unstructured":"Hong, F., Guzov, V., Kim, H.J.: Egolm: multi-modal language model of egocentric motions. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 5344\u20135354 (2025)","DOI":"10.1109\/CVPR52734.2025.00503"},{"key":"10_CR12","unstructured":"HuggingFace: LLaVA-OneVision documentation (2024). https:\/\/huggingface.co\/docs\/transformers\/en\/model_doc\/llava_onevision, Accessed 24 Apr 2026"},{"key":"10_CR13","doi-asserted-by":"crossref","unstructured":"Jia, W., Liu, M., Rehg, J.M.: Generative adversarial network for future hand segmentation from egocentric video. In: ECCV 2022, pp. 639\u2013656 (2022)","DOI":"10.1007\/978-3-031-19778-9_37"},{"key":"10_CR14","doi-asserted-by":"crossref","unstructured":"Kim, S.: Palm: predicting actions through language models. In: Proceedings of the 18th European Conference on Computer Vision (ECCV), pp. 140\u2013158 (2024)","DOI":"10.1007\/978-3-031-73007-8_9"},{"key":"10_CR15","doi-asserted-by":"crossref","unstructured":"Land, M., Mennie, N., Rusted, J.: The roles of vision and eye movements in the control of activities of daily living. Perception 28, 1311\u201328 (02 1999)","DOI":"10.1068\/p2935"},{"key":"10_CR16","unstructured":"Li, B.: Llava-onevision: Easy visual task transfer, arXiv:2408.03326 (2024)"},{"key":"10_CR17","unstructured":"Li, Y., Liu, M., Rehg, J.M.: In the eye of the beholder: Gaze and actions in first person video. arXiv:2006.00626 (2020)"},{"key":"10_CR18","doi-asserted-by":"crossref","unstructured":"Liu, M., Tang, S., Li, Y.: Forecasting human-object interaction: joint prediction of motor attention and actions in first person video, arXiv:1911.10967 (2020)","DOI":"10.1007\/978-3-030-58452-8_41"},{"key":"10_CR19","doi-asserted-by":"crossref","unstructured":"Ma, J., Bao, W., Xu, J.: Novel diffusion models for multimodal 3d hand trajectory prediction. In: 2025 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), pp. 2408\u20132415 (2025)","DOI":"10.1109\/IROS60139.2025.11246803"},{"key":"10_CR20","doi-asserted-by":"crossref","unstructured":"Makansi, O.: Multimodal future localization and emergence prediction for objects in egocentric view with a reachability prior. In: IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 4353\u20134362 (2020)","DOI":"10.1109\/CVPR42600.2020.00441"},{"key":"10_CR21","doi-asserted-by":"crossref","unstructured":"Mazzamuto, M., Ragusa, F., Furnari, A., Farinella, G.M.: Learning to detect attended objects in cultural sites with gaze signals and weak object supervision. J. Comput. Cult. Herit 17(3) (2024)","DOI":"10.1145\/3647999"},{"key":"10_CR22","doi-asserted-by":"crossref","unstructured":"Nagarajan, T., Feichtenhofer, C., Grauman, K.: Grounded human-object interaction hotspots from video. In: IEEE\/CVF International Conference on Computer Vision (ICCV), pp. 8687\u20138696 (2019)","DOI":"10.1109\/ICCV.2019.00878"},{"key":"10_CR23","doi-asserted-by":"crossref","unstructured":"Park, H.S., Hwang, J.J.: Egocentric future localization. In: IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 4697\u20134705 (2016)","DOI":"10.1109\/CVPR.2016.508"},{"key":"10_CR24","doi-asserted-by":"crossref","unstructured":"Pasca, R.G., Gavryushin, A., Hamza, M.: Summarize the past to predict the future: natural language descriptions of context boost multimodal object interaction anticipation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 18286\u201318296 (2024)","DOI":"10.1109\/CVPR52733.2024.01731"},{"key":"10_CR25","doi-asserted-by":"crossref","unstructured":"Perrett, T., et al.: Hd-epic: a highly-detailed egocentric video dataset. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 23901\u201323913 (2025)","DOI":"10.1109\/CVPR52734.2025.02226"},{"key":"10_CR26","doi-asserted-by":"crossref","unstructured":"Pirsiavash, H.: Detecting activities of daily living in first-person camera views. In: IEEE 2012 Conference on Computer Vision and Pattern Recognition (CVPR), pp. 2847\u20132854 (2012)","DOI":"10.1109\/CVPR.2012.6248010"},{"key":"10_CR27","doi-asserted-by":"publisher","first-page":"4880","DOI":"10.1007\/s11263-024-02095-7","volume":"132","author":"C Plizzari","year":"2024","unstructured":"Plizzari, C., Goletto, G., Furnari, A., et al.: An outlook into the future of egocentric vision. Int. J. Comput. Vision 132, 4880\u20134936 (2024)","journal-title":"Int. J. Comput. Vision"},{"key":"10_CR28","doi-asserted-by":"crossref","unstructured":"Plizzari, C.: Omnia de egotempo: benchmarking temporal understanding of multi-modal llms in egocentric videos. In: IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 24129\u201324138 (2025)","DOI":"10.1109\/CVPR52734.2025.02247"},{"key":"10_CR29","doi-asserted-by":"crossref","unstructured":"Ragusa, F., Farinella, G.M., Furnari, A.: Stillfast: an end-to-end approach for short-term object interaction anticipation. In: IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), pp. 3636\u20133645 (2023)","DOI":"10.1109\/CVPRW59228.2023.00371"},{"key":"10_CR30","doi-asserted-by":"crossref","unstructured":"Ragusa, F., Furnari, A., Farinella, G.M.: Meccano: A multimodal egocentric dataset for humans behavior understanding in the industrial-like domain. Comput. Vis. Image Underst. (CVIU) (2023)","DOI":"10.1016\/j.cviu.2023.103764"},{"key":"10_CR31","doi-asserted-by":"crossref","unstructured":"Ragusa, F., Furnari, A., Livatino, S., Farinella, G.M.: The meccano dataset: Understanding human-object interactions from egocentric videos in an industrial-like domain. In: Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision (WACV), pp. 1569\u20131578 (2021)","DOI":"10.1109\/WACV48630.2021.00161"},{"key":"10_CR32","doi-asserted-by":"crossref","unstructured":"Ragusa, F., et al.: Ego-extra: video-language egocentric dataset for expert-trainee assistance. arXiv:2512.13238 (2025)","DOI":"10.1109\/WACV61042.2026.00432"},{"key":"10_CR33","doi-asserted-by":"crossref","unstructured":"Ragusa, F.: Enigma-51: towards a fine-grained understanding of human behavior in industrial scenarios. In: IEEE\/CVF Winter Conference on Applications of Computer Vision (WACV), pp. 4537\u20134547 (2024)","DOI":"10.1109\/WACV57701.2024.00449"},{"issue":"1","key":"10_CR34","doi-asserted-by":"publisher","DOI":"10.1049\/cvi2.12342","volume":"19","author":"I Rodin","year":"2025","unstructured":"Rodin, I., Furnari, A., Farinella, G.M.: Egocentric action anticipation from untrimmed videos. IET Comput. Vision 19(1), e12342 (2025)","journal-title":"IET Comput. Vision"},{"key":"10_CR35","doi-asserted-by":"crossref","unstructured":"Roy, D., Rajendiran, R., Fernando, B.: Interaction region visual transformer for egocentric action anticipation. In: 2024 IEEE\/CVF Winter Conference on Applications of Computer Vision (WACV), pp. 6726\u20136736 (2024)","DOI":"10.1109\/WACV57701.2024.00660"},{"key":"10_CR36","unstructured":"Taluzzi, A.: From pixels to graphs: using scene and knowledge graphs for hd-epic vqa challenge arXiv:2506.08553 (2025)"},{"key":"10_CR37","doi-asserted-by":"publisher","first-page":"61767","DOI":"10.1109\/ACCESS.2024.3395282","volume":"12","author":"S Thakur","year":"2024","unstructured":"Thakur, S., Beyan, C., Morerio, P., Murino, V., Bue, A.D.: Anticipating next active objects for egocentric videos. IEEE Access 12, 61767\u201361779 (2024)","journal-title":"IEEE Access"},{"key":"10_CR38","unstructured":"Vaswani, A., Shazeer, N., Parmar, N.: Attention is all you need. In: Advances in Neural Information Processing Systems, vol. 30 (2017)"},{"key":"10_CR39","doi-asserted-by":"publisher","first-page":"2949","DOI":"10.1109\/TMM.2025.3557615","volume":"27","author":"B Wang","year":"2025","unstructured":"Wang, B., Tian, Y., Wang, S., Yang, L.: Multimodal large models are effective action anticipators. IEEE Trans. Multimedia 27, 2949\u20132960 (2025)","journal-title":"IEEE Trans. Multimedia"},{"key":"10_CR40","unstructured":"Wu, J., Zhang, Z., Xia, Y.: Visual prompting in multimodal large language models: A survey, arXiv:2409.15310 (2024)"},{"key":"10_CR41","unstructured":"Yang, J., Zhang, H., Li, F.: Set-of-mark prompting unleashes extraordinary visual grounding in gpt-4v, arXiv:2310.11441 (2023)"},{"key":"10_CR42","unstructured":"Yang, S.: Optimizing Multimodal LLMs for Egocentric Video Understanding: A Solution for the HD-EPIC VQA Challenge (2025). arXiv:2601.10228"},{"key":"10_CR43","doi-asserted-by":"crossref","unstructured":"Yarbus, A.L.: Eye Movements and Vision. Springer, US, New York (1967)","DOI":"10.1007\/978-1-4899-5379-7"},{"key":"10_CR44","doi-asserted-by":"crossref","unstructured":"Zhang, M., Ma, K.T.: Deep future gaze: gaze anticipation on egocentric videos using adversarial networks. In: IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 3539\u20133548 (2017)","DOI":"10.1109\/CVPR.2017.377"},{"key":"10_CR45","unstructured":"Zhang, Y., et\u00a0al.: HD-EPIC VQA benchmark result using Qwen2.5VL and LLaVA-Video (2025). https:\/\/hd-epic.github.io\/assets\/reports\/2025\/HD-EPIC-R3.pdf Accessed: 24 Apr 2026"},{"key":"10_CR46","doi-asserted-by":"crossref","unstructured":"Zhou, Y., Liu, L., Gou, C.: Learning from observer gaze: zero-shot attention prediction oriented by human-object interaction recognition. In: 2024 IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 28390\u201328400 (2024)","DOI":"10.1109\/CVPR52733.2024.02682"}],"container-title":["Lecture Notes in Computer Science","Pattern Recognition"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-31404-8_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,2]],"date-time":"2026-08-02T04:56:27Z","timestamp":1785646587000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-31404-8_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,3]]},"ISBN":["9783032314031","9783032314048"],"references-count":46,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-31404-8_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,8,3]]},"assertion":[{"value":"3 August 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","label":"Disclosure of Interests","group":{"name":"EthicsHeading","label":"Ethics"}},{"value":"ICPR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Pattern Recognition","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lyon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 August 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 August 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icpr2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/icpr2026.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}