{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,21]],"date-time":"2026-06-21T03:22:40Z","timestamp":1782012160905,"version":"3.54.5"},"reference-count":40,"publisher":"Elsevier BV","issue":"7","license":[{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,11,1]],"date-time":"2026-11-01T00:00:00Z","timestamp":1793491200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100004543","name":"China Scholarship Council","doi-asserted-by":"publisher","award":["202506250074"],"award-info":[{"award-number":["202506250074"]}],"id":[{"id":"10.13039\/501100004543","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100010880","name":"State Grid Corporation of China","doi-asserted-by":"publisher","award":["52270025000T-438-ZN"],"award-info":[{"award-number":["52270025000T-438-ZN"]}],"id":[{"id":"10.13039\/501100010880","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information Processing & Management"],"published-print":{"date-parts":[[2026,11]]},"DOI":"10.1016\/j.ipm.2026.104799","type":"journal-article","created":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T06:58:23Z","timestamp":1775804303000},"page":"104799","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"PA","title":["From Perception to Comprehension: A cognitive hierarchy for multimodal relation extraction"],"prefix":"10.1016","volume":"63","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1083-3837","authenticated-orcid":false,"given":"Zechen","family":"Meng","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mankun","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tianyi","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jian","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruiguo","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ying","family":"He","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5169-4829","authenticated-orcid":false,"given":"Mei","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.ipm.2026.104799_b1","series-title":"Findings of the association for computational linguistics","first-page":"1607","article-title":"Good visual guidance make a better extractor: Hierarchical visual prefix for multimodal entity and relation extraction","author":"Chen","year":"2022"},{"key":"10.1016\/j.ipm.2026.104799_b2","first-page":"1274","article-title":"Enhancing multimodal entity and relation extraction with variational information bottleneck","volume":"32","author":"Cui","year":"2024","journal-title":"IEEE ACM Transactions on Audio Speech and Language Processing"},{"key":"10.1016\/j.ipm.2026.104799_b3","series-title":"Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, NAACL-HLT 2019, minneapolis, MN, USA, June 2-7, 2019, volume 1 (long and short papers)","first-page":"4171","article-title":"BERT: pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"key":"10.1016\/j.ipm.2026.104799_b4","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2024.125608","article-title":"CE-DCVSI: multimodal relational extraction based on collaborative enhancement of dual-channel visual semantic information","volume":"262","author":"Gong","year":"2025","journal-title":"Expert Systems with Applications"},{"key":"10.1016\/j.ipm.2026.104799_b5","series-title":"Proceedings of the 2018 conference on empirical methods in natural language processing, Brussels, Belgium, October 31 - November 4, 2018","first-page":"2236","article-title":"Hierarchical relation extraction with coarse-to-fine grained attention","author":"Han","year":"2018"},{"issue":"1","key":"10.1016\/j.ipm.2026.104799_b6","doi-asserted-by":"crossref","DOI":"10.1016\/j.ipm.2024.103875","article-title":"The more quality information the better: Hierarchical generation of multi-evidence alignment and fusion model for multimodal entity and relation extraction","volume":"62","author":"He","year":"2025","journal-title":"Information Processing & Management"},{"key":"10.1016\/j.ipm.2026.104799_b7","doi-asserted-by":"crossref","unstructured":"He, L., Wang, H., Cao, Y., Wu, Z., Zhang, J., & Dai, X. (2023). More: A multimodal object-entity relation extraction dataset with a benchmark evaluation. In Proceedings of the 31st ACM international conference on multimedia (pp. 4564\u20134573). http:\/\/dx.doi.org\/10.1145\/3581783.3612209.","DOI":"10.1145\/3581783.3612209"},{"key":"10.1016\/j.ipm.2026.104799_b8","series-title":"2016 IEEE conference on computer vision and pattern recognition","first-page":"770","article-title":"Deep residual learning for image recognition","author":"He","year":"2016"},{"key":"10.1016\/j.ipm.2026.104799_b9","series-title":"Proceedings of the 31st ACM international conference on multimedia","first-page":"5185","article-title":"Prompt me up: Unleashing the power of alignments for multimodal entity and relation extraction","author":"Hu","year":"2023"},{"key":"10.1016\/j.ipm.2026.104799_b10","series-title":"Proceedings of the 61st annual meeting of the association for computational linguistics (volume 2: short papers)","first-page":"303","article-title":"Multimodal relation extraction with cross-modal retrieval and synthesis","author":"Hu","year":"2023"},{"issue":"3","key":"10.1016\/j.ipm.2026.104799_b11","doi-asserted-by":"crossref","DOI":"10.1016\/j.ipm.2024.104033","article-title":"A knowledge-enhanced network for joint multimodal entity-relation extraction","volume":"62","author":"Huang","year":"2025","journal-title":"Information Processing & Management"},{"key":"10.1016\/j.ipm.2026.104799_b12","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2025.113770","article-title":"ES-MRE: evidence subgraph enhanced reasoning for multimodal relation extraction","volume":"325","author":"Huang","year":"2025","journal-title":"Knowledge-Based Systems"},{"key":"10.1016\/j.ipm.2026.104799_b13","first-page":"16254","article-title":"On analyzing the role of image for visual-enhanced relation extraction (student abstract)","author":"Li","year":"2023"},{"key":"10.1016\/j.ipm.2026.104799_b14","series-title":"2025 IEEE international conference on acoustics, speech and signal processing","first-page":"1","article-title":"DFMA: adaptive dual fusion for multimodal relation extraction with mutual attention","author":"Li","year":"2025"},{"key":"10.1016\/j.ipm.2026.104799_b15","series-title":"International conference on machine learning","first-page":"12888","article-title":"BLIP: bootstrapping language-image pre-training for unified vision-language understanding and generation","volume":"vol. 162","author":"Li","year":"2022"},{"key":"10.1016\/j.ipm.2026.104799_b16","series-title":"Web information systems and applications - 21st international conference, WISA 2024, yinchuan, China, August 2-4, 2024, proceedings","first-page":"364","article-title":"Instruction tuning large language models for multimodal relation extraction using lora","volume":"vol. 14883","author":"Li","year":"2024"},{"key":"10.1016\/j.ipm.2026.104799_b17","series-title":"Computer vision - ECCV 2020 - 16th European conference, glasgow, UK, August 23-28, 2020, proceedings, part XXX","first-page":"121","article-title":"Oscar: Object-semantics aligned pre-training for vision-language tasks","volume":"vol. 12375","author":"Li","year":"2020"},{"key":"10.1016\/j.ipm.2026.104799_b18","series-title":"Proceedings of the 2021 conference of the North American chapter of the association for computational linguistics: human language technologies, NAACL-HLT 2021, online, June 6-11, 2021","first-page":"5339","article-title":"Unsupervised vision-and-language pre-training without parallel images and captions","author":"Li","year":"2021"},{"key":"10.1016\/j.ipm.2026.104799_b19","series-title":"Proceedings of the 55th annual meeting of the association for computational linguistics, ACL 2017, vancouver, Canada, July 30 - August 4, volume 1: long papers","first-page":"34","article-title":"Neural relation extraction with multi-lingual attention","author":"Lin","year":"2017"},{"key":"10.1016\/j.ipm.2026.104799_b20","series-title":"Proceedings of the ACM on web conference 2024","first-page":"4283","article-title":"Multimodal relation extraction via a mixture of hierarchical visual context learners","author":"Liu","year":"2024"},{"key":"10.1016\/j.ipm.2026.104799_b21","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2025.114023","article-title":"Hierarchical differential attention for multimodal relation extraction","volume":"327","author":"Liu","year":"2025","journal-title":"Knowledge-Based Systems"},{"key":"10.1016\/j.ipm.2026.104799_b22","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2025.130290","article-title":"Dual-perspective hypergraph learning network for multimodal entity and relation extraction","volume":"300","author":"Liu","year":"2026","journal-title":"Expert Systems with Applications"},{"key":"10.1016\/j.ipm.2026.104799_b23","series-title":"Proceedings of the 2018 conference of the North American chapter of the association for computational linguistics: human language technologies, NAACL-HLT 2018, new orleans, louisiana, USA, June 1-6, 2018, volume 1 (long papers)","first-page":"2227","article-title":"Deep contextualized word representations","author":"Peters","year":"2018"},{"key":"10.1016\/j.ipm.2026.104799_b24","series-title":"Proceedings of the 38th international conference on machine learning, ICML 2021, 18-24 July 2021, virtual event","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume":"vol. 139","author":"Radford","year":"2021"},{"key":"10.1016\/j.ipm.2026.104799_b25","series-title":"Proceedings of the 58th annual meeting of the association for computational linguistics, ACL 2020, online, July 5-10, 2020","first-page":"4498","article-title":"Improving multi-hop question answering over knowledge graphs using knowledge base embeddings","author":"Saxena","year":"2020"},{"key":"10.1016\/j.ipm.2026.104799_b26","series-title":"27th international conference on computer supported cooperative work in design","first-page":"2491","article-title":"Watch and read! a visual relation-aware and textual evidence enhanced model for multimodal relation extraction","author":"Shen","year":"2024"},{"key":"10.1016\/j.ipm.2026.104799_b27","first-page":"19062","article-title":"UMIE: unified multimodal information extraction with instruction tuning","author":"Sun","year":"2024"},{"key":"10.1016\/j.ipm.2026.104799_b28","series-title":"2020 IEEE\/CVF conference on computer vision and pattern recognition","first-page":"3713","article-title":"Unbiased scene graph generation from biased training","author":"Tang","year":"2020"},{"key":"10.1016\/j.ipm.2026.104799_b29","first-page":"111","article-title":"M3S: scene graph driven multi-granularity multi-task learning for multi-modal NER","volume":"31","author":"Wang","year":"2023","journal-title":"IEEE ACM Transactions on Audio Speech and Language Processing"},{"key":"10.1016\/j.ipm.2026.104799_b30","series-title":"Proceedings of the 61st annual meeting of the association for computational linguistics (volume 1: long papers)","first-page":"14734","article-title":"Information screening whilst exploiting! multimodal relation extraction with feature denoising and multimodal topic modeling","author":"Wu","year":"2023"},{"key":"10.1016\/j.ipm.2026.104799_b31","series-title":"MM \u201920: the 28th ACM international conference on multimedia, virtual event \/ seattle, WA, USA, October 12-16, 2020","first-page":"1038","article-title":"Multimodal representation with embedded visual guiding objects for named entity recognition in social media posts","author":"Wu","year":"2020"},{"key":"10.1016\/j.ipm.2026.104799_b32","series-title":"2019 IEEE\/CVF international conference on computer vision","first-page":"4682","article-title":"A fast and accurate one-stage approach to visual grounding","author":"Yang","year":"2019"},{"key":"10.1016\/j.ipm.2026.104799_b33","series-title":"Proceedings of the 31st international conference on computational linguistics","first-page":"978","article-title":"Enhancing multi-modal relation extraction with reinforcement learning guided graph diffusion framework","author":"Yang","year":"2025"},{"key":"10.1016\/j.ipm.2026.104799_b34","series-title":"Proceedings of the 58th annual meeting of the association for computational linguistics, ACL 2020, online, July 5-10, 2020","first-page":"3342","article-title":"Improving multimodal named entity recognition via entity span detection with unified multimodal transformer","author":"Yu","year":"2020"},{"issue":"2","key":"10.1016\/j.ipm.2026.104799_b35","doi-asserted-by":"crossref","first-page":"735","DOI":"10.1007\/s11280-019-00765-y","article-title":"A relationship extraction method for domain knowledge graph construction","volume":"23","author":"Yu","year":"2020","journal-title":"World Wide Web"},{"key":"10.1016\/j.ipm.2026.104799_b36","series-title":"Proceedings of the 63rd annual meeting of the association for computational linguistics (volume 1: long papers)","first-page":"26770","article-title":"Exploring multimodal relation extraction of hierarchical tabular data with multi-task learning","author":"Zhang","year":"2025"},{"key":"10.1016\/j.ipm.2026.104799_b37","first-page":"14347","article-title":"Multi-modal graph fusion for named entity recognition with targeted visual guidance","author":"Zhang","year":"2021"},{"key":"10.1016\/j.ipm.2026.104799_b38","series-title":"AAAI-25, sponsored by the association for the advancement of artificial intelligence, February 25 - March 4, 2025, philadelphia, PA, USA","first-page":"26003","article-title":"Prototype-guided multimodal relation extraction based on entity attributes","author":"Zhang","year":"2025"},{"key":"10.1016\/j.ipm.2026.104799_b39","series-title":"MM \u201921: ACM multimedia conference, virtual event, China, October 20 - 24, 2021","first-page":"5298","article-title":"Multimodal relation extraction with efficient graph alignment","author":"Zheng","year":"2021"},{"key":"10.1016\/j.ipm.2026.104799_b40","series-title":"2021 IEEE international conference on multimedia and expo","first-page":"1","article-title":"MNRE: a challenge multimodal dataset for neural relation extraction with visual evidence in social media posts","author":"Zheng","year":"2021"}],"container-title":["Information Processing & Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457326001901?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0306457326001901?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,6,20]],"date-time":"2026-06-20T11:23:15Z","timestamp":1781954595000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0306457326001901"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,11]]},"references-count":40,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2026,11]]}},"alternative-id":["S0306457326001901"],"URL":"https:\/\/doi.org\/10.1016\/j.ipm.2026.104799","relation":{},"ISSN":["0306-4573"],"issn-type":[{"value":"0306-4573","type":"print"}],"subject":[],"published":{"date-parts":[[2026,11]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"From Perception to Comprehension: A cognitive hierarchy for multimodal relation extraction","name":"articletitle","label":"Article Title"},{"value":"Information Processing & Management","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.ipm.2026.104799","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"104799"}}