{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,14]],"date-time":"2026-02-14T10:23:06Z","timestamp":1771064586811,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,13]],"date-time":"2024-05-13T00:00:00Z","timestamp":1715558400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,13]]},"DOI":"10.1145\/3589334.3645506","type":"proceedings-article","created":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T07:08:13Z","timestamp":1715152093000},"page":"2042-2051","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Using Model Calibration to Evaluate Link Prediction in Knowledge Graphs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-2849-0181","authenticated-orcid":false,"given":"Aishwarya","family":"Rao","sequence":"first","affiliation":[{"name":"Rochester Institute of Technology, Rochester, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1521-0735","authenticated-orcid":false,"given":"Narayanan Asuri","family":"Krishnan","sequence":"additional","affiliation":[{"name":"Rochester Institute of Technology, Rochester, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5770-950X","authenticated-orcid":false,"given":"Carlos R.","family":"Rivero","sequence":"additional","affiliation":[{"name":"Rochester Institute of Technology, Rochester, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,5,13]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Ralph Abboud .Ismail. Ilkan Ceylan Thomas Lukasiewicz and Tommaso Salvatori. 2020. BoxE: A Box Embedding Model for Knowledge Base Completion. In NeurIPS. 9649--9661."},{"key":"e_1_3_2_2_2_1","volume-title":"Qingheng Zhang, Wei Hu, and Chengkai Li.","author":"Akrami Farahnaz","year":"2020","unstructured":"Farahnaz Akrami, Mohammed Samiul Saeef, Qingheng Zhang, Wei Hu, and Chengkai Li. 2020. Realistic Re-evaluation of Knowledge Graph Completion Methods: An Experimental Study. In SIGMOD. 1995--2010."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"Mehdi Ali Max Berrendorf Mikhail Galkin Veronika Thost Tengfei Ma Volker Tresp and Jens Lehmann. 2022. Improving Inductive Link Prediction Using Hyper-Relational Facts (Extended Abstract). In IJCAI. 5259--5263.","DOI":"10.24963\/ijcai.2022\/731"},{"key":"e_1_3_2_2_4_1","volume-title":"Rivero","author":"Bansal Iti","year":"2020","unstructured":"Iti Bansal, Sudhanshu Tiwari, and Carlos R. Rivero. 2020. The Impact of Negative Triple Generation Strategies and Anomalies on Knowledge Graph Completion. In CIKM. 45--54."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"crossref","unstructured":"Anson Bastos Kuldeep Singh Abhishek Nadgeri Johannes Hoffart Manish Singh and Toyotaro Suzumura. 2023. Can Persistent Homology provide an efficient alternative for Evaluation of Knowledge Graph Completion Methods?. In TheWebConf. 2455--2466.","DOI":"10.1145\/3543507.3583308"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1376616.1376746"},{"key":"e_1_3_2_2_7_1","volume-title":"Hamilton","author":"Bonner Stephen","year":"2021","unstructured":"Stephen Bonner, Ian P. Barrett, Cheng Ye, Rowan Swiers, Ola Engkvist, and William L. Hamilton. 2021. Understanding the Performance of Knowledge Graph Embeddings in Drug Discovery. CoRR , Vol. abs\/2105.10488 (2021)."},{"key":"e_1_3_2_2_8_1","volume-title":"KDD.","author":"Bordes Antoine","year":"1967","unstructured":"Antoine Bordes and Evgeniy Gabrilovich. 2014. Constructing and mining web-scale knowledge graphs: KDD 2014 tutorial. In KDD. 1967."},{"key":"e_1_3_2_2_9_1","volume-title":"Jason Weston, and Oksana Yakhnenko.","author":"Bordes Antoine","year":"2013","unstructured":"Antoine Bordes, Nicolas Usunier, Alberto Garc'i a-Dur\u00e1 n, Jason Weston, and Oksana Yakhnenko. 2013. Translating Embeddings for Modeling Multi-relational Data. In NeurIPS. 2787--2795."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"crossref","unstructured":"Xuelu Chen Muhao Chen Weijia Shi Yizhou Sun and Carlo Zaniolo. 2019. Embedding Uncertain Knowledge Graphs. In AAAI. 3363--3370.","DOI":"10.1609\/aaai.v33i01.33013363"},{"key":"e_1_3_2_2_11_1","volume-title":"NAR","volume":"47","author":"The UniProt Consortium","year":"2018","unstructured":"The UniProt Consortium. 2018. UniProt: A worldwide hub of protein knowledge. NAR, Vol. 47, D1 (11 2018), D506--D515."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"crossref","unstructured":"Tim Dettmers Pasquale Minervini Pontus Stenetorp and Sebastian Riedel. 2018. Convolutional 2D Knowledge Graph Embeddings. In AAAI. 1811--1818.","DOI":"10.1609\/aaai.v32i1.11573"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623623"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"crossref","unstructured":"Xin Luna Dong. 2019. Building a Broad Knowledge Graph for Products. In ICDE. 25.","DOI":"10.1109\/ICDE.2019.00010"},{"key":"e_1_3_2_2_15_1","volume-title":"Draper and Harry Smith","author":"Norman","year":"1998","unstructured":"Norman R. Draper and Harry Smith. 1998. Applied Regression Analysis, 3$^rd$ Edition. Wiley."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"crossref","unstructured":"Takuma Ebisu and Ryutaro Ichise. 2018. TorusE: Knowledge Graph Embedding on a Lie Group. In AAAI. 1819--1826.","DOI":"10.1609\/aaai.v32i1.11538"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.7554\/eLife.26726"},{"key":"e_1_3_2_2_19_1","volume-title":"Roberto Navigli, Sebastian Neumaier, Axel-Cyrille Ngonga Ngomo, Axel Polleres, Sabbir M. Rashid, Anisa Rula, Lukas Schmelzeisen, Juan F. Sequeda, Steffen Staab, and Antoine Zimmermann.","author":"Hogan Aidan","year":"2022","unstructured":"Aidan Hogan, Eva Blomqvist, Michael Cochez, Claudia d'Amato, Gerard de Melo, Claudio Gutierrez, Sabrina Kirrane, Jos\u00e9 Emilio Labra Gayo, Roberto Navigli, Sebastian Neumaier, Axel-Cyrille Ngonga Ngomo, Axel Polleres, Sabbir M. Rashid, Anisa Rula, Lukas Schmelzeisen, Juan F. Sequeda, Steffen Staab, and Antoine Zimmermann. 2022. Knowledge Graphs. ACM Comput. Surv. , Vol. 54, 4 (2022), 71:1--71:37."},{"key":"e_1_3_2_2_20_1","volume-title":"Gyori","author":"Hoyt Charles Tapley","year":"2022","unstructured":"Charles Tapley Hoyt, Max Berrendorf, Mikhail Galkin, Volker Tresp, and Benjamin M. Gyori. 2022. A Unified Framework for Rank-based Evaluation Metrics for Link Prediction in Knowledge Graphs. CoRR , Vol. abs\/2203.07544 (2022)."},{"key":"e_1_3_2_2_21_1","volume-title":"Rivero","author":"Krishnan Narayanan Asuri","year":"2023","unstructured":"Narayanan Asuri Krishnan and Carlos R. Rivero. 2023. A Model-Agnostic Method to Interpret Link Prediction Evaluation of Knowledge Graph Embeddings. In CIKM. 1107--1116."},{"key":"e_1_3_2_2_22_1","volume-title":"Telmo de Menezes e Silva Filho, and Peter A. Flach","author":"Kull Meelis","year":"2017","unstructured":"Meelis Kull, Telmo de Menezes e Silva Filho, and Peter A. Flach. 2017. Beta calibration: a well-founded and easily implemented improvement on logistic calibration for binary classifiers. In AISTATS (Proceedings of Machine Learning Research, Vol. 54). 623--631."},{"key":"e_1_3_2_2_23_1","volume-title":"Suchanek","author":"Mahdisoltani Farzaneh","year":"2015","unstructured":"Farzaneh Mahdisoltani, Joanna Biega, and Fabian M. Suchanek. 2015. YAGO3: A Knowledge Base from Multilingual Wikipedias. In CIDR."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"e_1_3_2_2_25_1","volume-title":"Poggio","author":"Nickel Maximilian","year":"2016","unstructured":"Maximilian Nickel, Lorenzo Rosasco, and Tomaso A. Poggio. 2016. Holographic Embeddings of Knowledge Graphs. In AAAI. 1955--1961."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1102351.1102430"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331166"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.3233\/SW-160218"},{"key":"e_1_3_2_2_29_1","unstructured":"Pouya Pezeshkpour Yifan Tian and Sameer Singh. 2020. Revisiting Evaluation of Knowledge Base Completion Models. In AKBC."},{"key":"e_1_3_2_2_30_1","volume-title":"Mitchell","author":"Platanios Emmanouil Antonios","year":"2020","unstructured":"Emmanouil Antonios Platanios, Abulhair Saparov, and Tom M. Mitchell. 2020. Jelly Bean World: A Testbed for Never-Ending Learning. In ICLR."},{"key":"e_1_3_2_2_31_1","volume-title":"Advances in Large Margin Classifiers, , Alexander J","author":"Platt John C.","unstructured":"John C. Platt. 1999. Probabilistic Outputs for Support Vector Machines and Comparisons to Regularized Likelihood Methods. In Advances in Large Margin Classifiers, , Alexander J. Smola, Peter Bartlett, Bernhard Sch\u00f6lkopf, and Dale Schuurmans (Eds.). MIT Press, Chapter 10, 61--74."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3424672"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-017-05778-z"},{"key":"e_1_3_2_2_34_1","unstructured":"Daniel Ruffinelli Samuel Broscheit and Rainer Gemulla. 2020. You CAN Teach an Old Dog New Tricks! On Training Knowledge Graph Embeddings. In ICLR."},{"key":"e_1_3_2_2_35_1","unstructured":"Afshin Sadeghi Hirra Malik Diego Collarana and Jens Lehmann. 2021. Relational Pattern Benchmarking on the Knowledge Graph Link Prediction Task. In NeurIPS Datasets and Benchmarks."},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"crossref","unstructured":"Tara Safavi Danai Koutra and Edgar Meij. 2020. Evaluating the Calibration of Knowledge Graph Embeddings for Trustworthy Link Prediction. In EMNLP. 8308--8321.","DOI":"10.18653\/v1\/2020.emnlp-main.667"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.softx.2023.101474"},{"key":"e_1_3_2_2_38_1","unstructured":"Tengwei Song Jie Luo and Lei Huang. 2021. Rot-Pro: Modeling Transitivity by Projection in Knowledge Graph Embedding. In NeurIPS. 24695--24706."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"crossref","unstructured":"Fabian M. Suchanek Gjergji Kasneci and Gerhard Weikum. 2007. Yago: a core of semantic knowledge. In WWW. 697--706.","DOI":"10.1145\/1242572.1242667"},{"key":"e_1_3_2_2_40_1","unstructured":"Zhiqing Sun Zhi-Hong Deng Jian-Yun Nie and Jian Tang. 2019. RotatE: Knowledge Graph Embedding by Relational Rotation in Complex Space. In ICLR Posters."},{"key":"e_1_3_2_2_41_1","unstructured":"Pedro Tabacof and Luca Costabello. 2020. Probability Calibration for Knowledge Graph Embedding Models. In ICLR."},{"key":"e_1_3_2_2_42_1","volume-title":"Rivero","author":"Tiwari Sudhanshu","year":"2021","unstructured":"Sudhanshu Tiwari, Iti Bansal, and Carlos R. Rivero. 2021. Revisiting the Evaluation Protocol of Knowledge Graph Completion Methods for Link Prediction. In TheWebConf. 809--820."},{"key":"e_1_3_2_2_43_1","volume-title":"Observed Versus Latent Features for Knowledge Base and Text Inference. In ACL Workshops. 57--66","author":"Toutanova Kristina","year":"2015","unstructured":"Kristina Toutanova and Danqi Chen. 2015. Observed Versus Latent Features for Knowledge Base and Text Inference. In ACL Workshops. 57--66."},{"key":"e_1_3_2_2_44_1","first-page":"2071","article-title":"Complex Embeddings for Simple Link Prediction","volume":"48","author":"Trouillon Th\u00e9","year":"2016","unstructured":"Th\u00e9 o Trouillon, Johannes Welbl, Sebastian Riedel, \u00c9 ric Gaussier, and Guillaume Bouchard. 2016. Complex Embeddings for Simple Link Prediction. In ICML, Vol. 48. 2071--2080.","journal-title":"ICML"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"crossref","unstructured":"Brian Walsh Sameh K. Mohamed and V'i t Nov\u00e1 cek. 2020. BioKG: A Knowledge Graph for Relational Learning On Biological Data. In CIKM. 3173--3180.","DOI":"10.1145\/3340531.3412776"},{"key":"e_1_3_2_2_46_1","first-page":"2724","article-title":"Knowledge Graph Embedding","volume":"29","author":"Wang Quan","year":"2017","unstructured":"Quan Wang, Zhendong Mao, Bin Wang, and Li Guo. 2017. Knowledge Graph Embedding: A Survey of Approaches and Applications. TKDE, Vol. 29, 12 (2017), 2724--2743.","journal-title":"A Survey of Approaches and Applications. TKDE"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"crossref","unstructured":"Wenhan Xiong Thien Hoang and William Yang Wang. 2017. DeepPath: A Reinforcement Learning Method for Knowledge Graph Reasoning. In EMNLP. 564--573.","DOI":"10.18653\/v1\/D17-1060"},{"key":"e_1_3_2_2_48_1","volume-title":"Decomposition of the Brier Score for Weighted Forecast-Verification Pairs. RMetS","author":"Young R. M. B.","unstructured":"R. M. B. Young. [n.,d.]. Decomposition of the Brier Score for Weighted Forecast-Verification Pairs. RMetS, Vol. 136, 650 ( [n.,d.]), 1364--1370."},{"key":"e_1_3_2_2_49_1","unstructured":"Shuai Zhang Yi Tay Lina Yao and Qi Liu. 2019. Quaternion Knowledge Graph Embeddings. In NeurIPS. 2731--2741."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"crossref","unstructured":"Zhanqiu Zhang Jianyu Cai Yongdong Zhang and Jie Wang. 2020. Learning Hierarchy-Aware Knowledge Graph Embeddings for Link Prediction. In AAAI. 3065--3072.","DOI":"10.1609\/aaai.v34i03.5701"}],"event":{"name":"WWW '24: The ACM Web Conference 2024","location":"Singapore Singapore","acronym":"WWW '24","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Web Conference 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645506","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589334.3645506","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T00:35:42Z","timestamp":1755822942000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589334.3645506"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,13]]},"references-count":50,"alternative-id":["10.1145\/3589334.3645506","10.1145\/3589334"],"URL":"https:\/\/doi.org\/10.1145\/3589334.3645506","relation":{},"subject":[],"published":{"date-parts":[[2024,5,13]]},"assertion":[{"value":"2024-05-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}