{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T01:44:47Z","timestamp":1773193487621,"version":"3.50.1"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2021,4,13]],"date-time":"2021-04-13T00:00:00Z","timestamp":1618272000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,4,13]],"date-time":"2021-04-13T00:00:00Z","timestamp":1618272000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61762061"],"award-info":[{"award-number":["61762061"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61763031"],"award-info":[{"award-number":["61763031"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004479","name":"Natural Science Foundation of Jiangxi Province","doi-asserted-by":"publisher","award":["20161ACB20004"],"award-info":[{"award-number":["20161ACB20004"]}],"id":[{"id":"10.13039\/501100004479","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Jiangxi Key Laboratory of Smart City","award":["20192BCD40002"],"award-info":[{"award-number":["20192BCD40002"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62076117"],"award-info":[{"award-number":["62076117"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2021,12]]},"DOI":"10.1007\/s10489-021-02304-7","type":"journal-article","created":{"date-parts":[[2021,4,13]],"date-time":"2021-04-13T03:43:46Z","timestamp":1618285426000},"page":"8768-8783","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Multimodal graph inference network for scene graph generation"],"prefix":"10.1007","volume":"51","author":[{"given":"Jingwen","family":"Duan","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2526-2181","authenticated-orcid":false,"given":"Weidong","family":"Min","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Deyu","family":"Lin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jianfeng","family":"Xu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xin","family":"Xiong","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,4,13]]},"reference":[{"key":"2304_CR1","doi-asserted-by":"crossref","unstructured":"Johnson J, Krishna R, Stark M, Li LJ, Shamma D, Bernstein M, Fei-Fei L (2015) Image retrieval using scene graphs. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3668\u20133678","DOI":"10.1109\/CVPR.2015.7298990"},{"key":"2304_CR2","unstructured":"Krizhevsky A, Sutskever I, Hinton GE (2012) Imagenet classification with deep convolutional neural networks. In: Advances in neural information processing systems, pp 1097\u20131105"},{"key":"2304_CR3","doi-asserted-by":"crossref","unstructured":"Lin TY, Doll\u00e1r P, Girshick R, He K, Hariharan B, Belongie S (2017) Feature pyramid networks for object detection. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 2117\u20132125","DOI":"10.1109\/CVPR.2017.106"},{"key":"2304_CR4","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S, Sun J (2016) Deep residual learning for image recognition. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"key":"2304_CR5","doi-asserted-by":"crossref","unstructured":"Xiong X, Min W, Zheng W, Liao P, Yang H, Wang S (2020) S3d-cnn: skeleton-based 3d consecutive-low-pooling neural network for fall detection. Applied Intelligence","DOI":"10.1007\/s10489-020-01751-y"},{"key":"2304_CR6","doi-asserted-by":"crossref","unstructured":"Qiu J, Dong Y, Ma H, Li J, Wang K, Tang J (2018) Network embedding as matrix factorization: Unifying deepwalk, line, pte, and node2vec. In: Proceedings of the eleventh ACM international conference on web search and data mining, 459\u2013467","DOI":"10.1145\/3159652.3159706"},{"issue":"5","key":"2304_CR7","doi-asserted-by":"publisher","first-page":"833","DOI":"10.1109\/TKDE.2018.2849727","volume":"31","author":"P Cui","year":"2018","unstructured":"Cui P, Wang X, Pei J, Zhu W (2018) A survey on network embedding. IEEE Trans Knowl Data Eng 31(5):833\u2013852","journal-title":"IEEE Trans Knowl Data Eng"},{"issue":"9","key":"2304_CR8","doi-asserted-by":"publisher","first-page":"1616","DOI":"10.1109\/TKDE.2018.2807452","volume":"30","author":"H Cai","year":"2018","unstructured":"Cai H, Zheng VW, Chang KCC (2018) A comprehensive survey of graph embedding: Problems, techniques, and applications. IEEE Trans Knowl Data Eng 30(9):1616\u20131637","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"2304_CR9","unstructured":"Li Y, Tarlow D, Brockschmidt M, Zemel R (2015) Gated graph sequence neural networks. arXiv:151105493"},{"issue":"1","key":"2304_CR10","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1109\/TNN.2008.2005605","volume":"20","author":"F Scarselli","year":"2008","unstructured":"Scarselli F, Gori M, Tsoi AC, Hagenbuchner M, Monfardini G (2008) The graph neural network model. IEEE Trans Neural Netw 20(1):61\u201380","journal-title":"IEEE Trans Neural Netw"},{"issue":"1","key":"2304_CR11","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1109\/TNN.2008.2005141","volume":"20","author":"F Scarselli","year":"2008","unstructured":"Scarselli F, Gori M, Tsoi AC, Hagenbuchner M, Monfardini G (2008) Computational capabilities of graph neural networks. IEEE Trans Neural Netw 20(1):81\u2013102","journal-title":"IEEE Trans Neural Netw"},{"issue":"8","key":"2304_CR12","doi-asserted-by":"publisher","first-page":"2117","DOI":"10.1109\/TMM.2019.2896516","volume":"21","author":"X Li","year":"2019","unstructured":"Li X, Jiang S (2019) Know more say less: Image captioning based on scene graphs. IEEE Trans Multimed 21(8):2117\u20132130","journal-title":"IEEE Trans Multimed"},{"key":"2304_CR13","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1016\/j.jvcir.2018.12.027","volume":"58","author":"N Xu","year":"2019","unstructured":"Xu N, Liu A-A, Liu J, Nie W, Su Y (2019) Scene graph captioner: Image captioning based on structural visual representation. J Vis Commun Image Represent 58:477\u2013485","journal-title":"J Vis Commun Image Represent"},{"key":"2304_CR14","doi-asserted-by":"crossref","unstructured":"Yang X, Tang K, Zhang H, Cai J (2019) Auto-encoding scene graphs for image captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, 10685\u201310694","DOI":"10.1109\/CVPR.2019.01094"},{"key":"2304_CR15","doi-asserted-by":"publisher","first-page":"115648","DOI":"10.1016\/j.image.2019.115648","volume":"80","author":"Y Xi","year":"2020","unstructured":"Xi Y, Zhang Y, Ding S, Wan S (2020) Visual question answering model based on visual relationship detection. Signal Process Image Commun 80:115648","journal-title":"Signal Process Image Commun"},{"key":"2304_CR16","doi-asserted-by":"crossref","unstructured":"Prabhu N, Venkatesh Babu R (2015) Attribute-graph: A graph based approach to image ranking. In: Proceedings of the IEEE international conference on computer vision, pp 1071\u20131079","DOI":"10.1109\/ICCV.2015.128"},{"key":"2304_CR17","doi-asserted-by":"crossref","unstructured":"Herzig R, Bar A, Xu H, Chechik G, Darrell T, Globerson A (2019) Learning canonical representations for scene graph to image generation. arXiv:191207414","DOI":"10.1007\/978-3-030-58574-7_13"},{"key":"2304_CR18","doi-asserted-by":"crossref","unstructured":"Chen T, Yu W, Chen R, Lin L (2019) Knowledge-embedded routing network for scene graph generation. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 6163\u20136171","DOI":"10.1109\/CVPR.2019.00632"},{"issue":"1","key":"2304_CR19","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","volume":"123","author":"R Krishna","year":"2017","unstructured":"Krishna R, Zhu Y, Groth O, Johnson J, Hata K, Kravitz J, Chen S, Kalantidis Y, Li LJ, Shamma DA et al (2017) Visual genome: Connecting language and vision using crowdsourced dense image annotations. Int J Comput Vis 123(1):32\u201373","journal-title":"Int J Comput Vis"},{"key":"2304_CR20","doi-asserted-by":"crossref","unstructured":"Zhang H, Kyaw Z, Chang S-F, Chua T-S (2017) Visual translation embedding network for visual relation detection. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 5532\u20135540","DOI":"10.1109\/CVPR.2017.331"},{"key":"2304_CR21","doi-asserted-by":"crossref","unstructured":"Wan H, Luo Y, Peng B, Zheng W-S (2018) Representation learning for scene graph completion via jointly structural and visual embedding. In: IJCAI, pp 949\u2013956","DOI":"10.24963\/ijcai.2018\/132"},{"key":"2304_CR22","doi-asserted-by":"crossref","unstructured":"Hung Z-S, Mallya A, Lazebnik S (2020) Contextual translation embedding for visual relationship detection and scene graph generation. IEEE Transactions on Pattern Analysis and Machine Intelligence","DOI":"10.1109\/TPAMI.2020.2992222"},{"key":"2304_CR23","doi-asserted-by":"crossref","unstructured":"Dai B, Zhang Y, Lin D (2017) Detecting visual relationships with deep relational networks. In: Proceedings of the IEEE conference on computer vision and Pattern recognition, pp 3076\u20133086","DOI":"10.1109\/CVPR.2017.352"},{"key":"2304_CR24","doi-asserted-by":"crossref","unstructured":"Xu D, Zhu Y, Choy CB, Fei-Fei L (2017) Scene graph generation by iterative message passing. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp 5410\u20135419","DOI":"10.1109\/CVPR.2017.330"},{"key":"2304_CR25","doi-asserted-by":"crossref","unstructured":"Li Y, Ouyang W, Zhou B, Wang K, Wang X (2017) Scene graph generation from objects, phrases and region captions. In: Proceedings of the IEEE international conference on computer vision, pp 1261\u20131270","DOI":"10.1109\/ICCV.2017.142"},{"key":"2304_CR26","doi-asserted-by":"crossref","unstructured":"Lu C, Krishna R, Bernstein M, Fei-Fei L (2016) Visual relationship detection with language priors. In: European conference on computer vision, Springer, pp 852\u2013869","DOI":"10.1007\/978-3-319-46448-0_51"},{"key":"2304_CR27","doi-asserted-by":"crossref","unstructured":"Zellers R, Yatskar M, Thomson S, Choi Y (2018) Neural motifs: Scene graph parsing with global context. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 5831\u20135840","DOI":"10.1109\/CVPR.2018.00611"},{"key":"2304_CR28","doi-asserted-by":"crossref","unstructured":"Gu J, Zhao H, Lin Z, Li S, Cai J, Ling M (2019) Scene graph generation with external knowledge and image reconstruction. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 1969\u20131978","DOI":"10.1109\/CVPR.2019.00207"},{"key":"2304_CR29","doi-asserted-by":"crossref","unstructured":"Li Y, Ouyang W, Zhou B, Shi J, Zhang C, Wang X (2018) Factorizable net: an efficient subgraph-based framework for scene graph generation. In: Proceedings of the european conference on computer vision (ECCV), pp 335\u2013351","DOI":"10.1007\/978-3-030-01246-5_21"},{"key":"2304_CR30","doi-asserted-by":"crossref","unstructured":"Yang J, Lu J, Lee S, Batra D, Parikh D (2018) Graph r-cnn for scene graph generation. In: Proceedings of the European conference on computer vision (ECCV), pp 670\u2013685","DOI":"10.1007\/978-3-030-01246-5_41"},{"issue":"4","key":"2304_CR31","doi-asserted-by":"publisher","first-page":"885","DOI":"10.1109\/TMM.2019.2934833","volume":"22","author":"C Deng","year":"2019","unstructured":"Deng C, Yang X, Nie F, Tao D (2019) Saliency detection via a multiple self-weighted graph-based manifold ranking. IEEE Trans Multimed 22(4):885\u2013896","journal-title":"IEEE Trans Multimed"},{"key":"2304_CR32","doi-asserted-by":"crossref","unstructured":"Li C, Tang H, Deng C, Zhan L, Liu W (2020) Vulnerability vs. reliability: Disentangled adversarial examples for cross-modal learning. In: Proceedings of the 26th ACM SIGKDD International conference on knowledge discovery & data mining, association for computing machinery, pp 421\u2013429","DOI":"10.1145\/3394486.3403084"},{"issue":"4","key":"2304_CR33","doi-asserted-by":"publisher","first-page":"1155","DOI":"10.1007\/s10489-019-01581-7","volume":"50","author":"W Guo","year":"2020","unstructured":"Guo W, Cai J, Wang S (2020) Unsupervised discriminative feature representation via adversarial auto-encoder. Appl Intell 50(4):1155\u20131171","journal-title":"Appl Intell"},{"key":"2304_CR34","doi-asserted-by":"publisher","first-page":"63373","DOI":"10.1109\/ACCESS.2019.2916887","volume":"7","author":"W Guo","year":"2019","unstructured":"Guo W, Wang J, Wang S (2019) Deep multimodal representation learning: A survey. IEEE Access 7:63373\u201363394","journal-title":"IEEE Access"},{"issue":"10","key":"2304_CR35","doi-asserted-by":"publisher","first-page":"2303","DOI":"10.1109\/TPAMI.2017.2753232","volume":"40","author":"Y Aytar","year":"2017","unstructured":"Aytar Y, Castrejon L, Vondrick C, Pirsiavash H, Torralba A (2017) Cross-modal scene networks. IEEE Trans Pattern Anal Mach Intell 40(10):2303\u20132314","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"2304_CR36","doi-asserted-by":"publisher","first-page":"3626","DOI":"10.1109\/TIP.2020.2963957","volume":"29","author":"D Xie","year":"2020","unstructured":"Xie D, Deng C, Li C, Liu X, Tao D (2020) Multi-task consistency-preserving adversarial hashing for cross-modal retrieval. IEEE Trans Image Process 29:3626\u20133637","journal-title":"IEEE Trans Image Process"},{"issue":"11","key":"2304_CR37","doi-asserted-by":"publisher","first-page":"5292","DOI":"10.1109\/TNNLS.2018.2793863","volume":"29","author":"E Yang","year":"2018","unstructured":"Yang E, Deng C, Li C, Liu W, Li J, Tao D (2018) Shared predictive cross-modal deep quantization. IEEE Trans Neural Netw Learn Syst 29(11):5292\u20135303","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2304_CR38","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1016\/j.cviu.2016.11.004","volume":"155","author":"S Wang","year":"2017","unstructured":"Wang S, Zhang H, Wang H (2017) Object co-segmentation via weakly supervised data fusion. Comput Vis Image Underst 155:43\u201354","journal-title":"Comput Vis Image Underst"},{"issue":"2","key":"2304_CR39","doi-asserted-by":"publisher","first-page":"352","DOI":"10.1109\/TPAMI.2017.2670560","volume":"40","author":"YG Jiang","year":"2017","unstructured":"Jiang YG, Wu Z, Wang J, Xue X, Chang SF (2017) Exploiting feature and class relationships in video categorization with regularized deep neural networks. IEEE Trans Pattern Anal Mach Intell 40 (2):352\u2013364","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"3","key":"2304_CR40","doi-asserted-by":"publisher","first-page":"663","DOI":"10.1007\/s10489-019-01540-2","volume":"50","author":"AD Vo","year":"2020","unstructured":"Vo AD, Nguyen QP, Ock CY (2020) Semantic and syntactic analysis in learning representation based on a sentiment analysis model. Appl Intell 50(3):663\u2013680","journal-title":"Appl Intell"},{"issue":"1","key":"2304_CR41","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1109\/JPROC.2015.2483592","volume":"104","author":"M Nickel","year":"2015","unstructured":"Nickel M, Murphy K, Tresp V, Gabrilovich E (2015) A review of relational machine learning for knowledge graphs. Proc IEEE 104(1):11\u201333","journal-title":"Proc IEEE"},{"key":"2304_CR42","doi-asserted-by":"crossref","unstructured":"Marino K, Salakhutdinov R, Gupta A (2016) The more you know: Using knowledge graphs for image classification. arXiv:161204844","DOI":"10.1109\/CVPR.2017.10"},{"key":"2304_CR43","doi-asserted-by":"crossref","unstructured":"Lee CW, Fang W, Yeh CK, Frank Wang YC (2018) Multi-label zero-shot learning with structured knowledge graphs. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 1576\u20131585","DOI":"10.1109\/CVPR.2018.00170"},{"key":"2304_CR44","doi-asserted-by":"crossref","unstructured":"Yang X, Deng C, Liu T, Tao D (2020) Heterogeneous graph attention network for unsupervised multiple-target domain adaptation. IEEE Transactions on Pattern Analysis and Machine Intelligence","DOI":"10.1109\/TPAMI.2020.3026079"},{"key":"2304_CR45","doi-asserted-by":"crossref","unstructured":"Pennington J, Socher R, Manning CD (2014) Glove: Global vectors for word representation. In: Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP), pp 1532\u20131543","DOI":"10.3115\/v1\/D14-1162"},{"key":"2304_CR46","unstructured":"Kipf TN, Welling M (2016) Semi-supervised classification with graph convolutional networks. arXiv:160902907"},{"key":"2304_CR47","doi-asserted-by":"crossref","unstructured":"Redmon J, Farhadi A (2017) Yolo9000: better, faster, stronger. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 7263\u20137271","DOI":"10.1109\/CVPR.2017.690"},{"key":"2304_CR48","doi-asserted-by":"crossref","unstructured":"Tang K, Niu Y, Huang J, Shi J, Zhang H (2020) Unbiased scene graph generation from biased training. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3716\u20133725","DOI":"10.1109\/CVPR42600.2020.00377"},{"key":"2304_CR49","doi-asserted-by":"crossref","unstructured":"Chen V, Varma P, Krishna R, Bernstein M, Re C, Fei-Fei L (2019) Scene graph prediction with limited labels. In: International conference on computer vision","DOI":"10.1109\/ICCVW.2019.00220"},{"key":"2304_CR50","unstructured":"Newell A, Deng J (2017) Pixels to graphs by associative embedding. In: Advances in neural information processing systems, pp 2171\u20132180"},{"key":"2304_CR51","doi-asserted-by":"crossref","unstructured":"Khademi M, Schulte O (2018) Dynamic gated graph neural networks for scene graph generation. In: Asian conference on computer vision, Springer, pp 669\u2013685","DOI":"10.1007\/978-3-030-20876-9_42"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-021-02304-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-021-02304-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-021-02304-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,11,9]],"date-time":"2021-11-09T05:08:11Z","timestamp":1636434491000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-021-02304-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,13]]},"references-count":51,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2021,12]]}},"alternative-id":["2304"],"URL":"https:\/\/doi.org\/10.1007\/s10489-021-02304-7","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,13]]},"assertion":[{"value":"2 March 2021","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 April 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}