{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,3]],"date-time":"2026-09-03T15:19:38Z","timestamp":1788448778968,"version":"build-2803163510"},"reference-count":42,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,5,1]],"date-time":"2025-05-01T00:00:00Z","timestamp":1746057600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,5,1]],"date-time":"2025-05-01T00:00:00Z","timestamp":1746057600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,2,24]],"date-time":"2025-02-24T00:00:00Z","timestamp":1740355200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100018693","name":"Horizon Europe","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100018693","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100021856","name":"Ministero dell&apos;Universit\u00e0 e della Ricerca","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100021856","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["clinicalkey.fr","clinicalkey.jp","clinicalkey.es","clinicalkey.com.au","clinicalkey.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["International Journal of Medical Informatics"],"published-print":{"date-parts":[[2025,5]]},"DOI":"10.1016\/j.ijmedinf.2025.105857","type":"journal-article","created":{"date-parts":[[2025,2,25]],"date-time":"2025-02-25T08:16:11Z","timestamp":1740471371000},"page":"105857","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":9,"special_numbering":"C","title":["RelAI: an automated approach to judge pointwise ML prediction reliability"],"prefix":"10.1016","volume":"197","author":[{"given":"Lorenzo","family":"Peracchio","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Giovanna","family":"Nicora","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Enea","family":"Parimbelli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tommaso Mario","family":"Buonocore","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eleonora","family":"Tavazzi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Roberto","family":"Bergamaschi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arianna","family":"Dagliati","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Riccardo","family":"Bellazzi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.ijmedinf.2025.105857_b0005","doi-asserted-by":"crossref","first-page":"807","DOI":"10.1016\/j.gie.2020.06.040","article-title":"History of artificial intelligence in medicine","volume":"92","author":"Kaul","year":"2020","journal-title":"Gastrointest. Endosc."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0010","unstructured":"R. Bommasani et al., \u2018On the Opportunities and Risks of Foundation Models\u2019, Jul. 12, 2022, arXiv: arXiv:2108.07258. doi: 10.48550\/arXiv.2108.07258."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0015","doi-asserted-by":"crossref","unstructured":"M. Moor et al., \u2018Foundation models for generalist medical artificial intelligence\u2019, Nature, vol. 616, no. 7956, Art. no. 7956, Apr. 2023, doi: 10.1038\/s41586-023-05881-4.","DOI":"10.1038\/s41586-023-05881-4"},{"issue":"2","key":"10.1016\/j.ijmedinf.2025.105857_b0020","doi-asserted-by":"crossref","first-page":"309","DOI":"10.1111\/cts.13175","article-title":"Artificial intelligence in clinical and translational science: Successes, challenges and opportunities","volume":"15","author":"Bernstam","year":"2022","journal-title":"Clin. Transl. Sci."},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0025","first-page":"1","article-title":"\u2018An operational guide to translational clinical machine learning in academic medical centers\u2019, npj Digit","volume":"7","author":"Poddar","year":"2024","journal-title":"Med.,"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0030","unstructured":"P. O. of the E. Union, \u2018The Assessment List for Trustworthy Artificial Intelligence (ALTAI) for self assessment.\u2019 Accessed: Sep. 07, 2022. [Online]. Available: http:\/\/op.europa.eu\/en\/publication-detail\/-\/publication\/73552fcd-f7c2-11ea-991b-01aa75ed71a1."},{"issue":"5","key":"10.1016\/j.ijmedinf.2025.105857_b0035","doi-asserted-by":"crossref","first-page":"1235","DOI":"10.1038\/s41591-024-02874-2","article-title":"New AI regulation in the EU seeks to reduce risk without assessing public benefit","volume":"30","author":"Prainsack","year":"2024","journal-title":"Nat. Med."},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0040","first-page":"1","article-title":"\u2018The EU passes the AI Act and its implications for digital medicine are unclear\u2019, npj Digit","volume":"7","author":"Gilbert","year":"2024","journal-title":"Med.,"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0045","doi-asserted-by":"crossref","DOI":"10.3389\/fdgth.2022.929508","article-title":"The silent trial - the bridge between bench-to-bedside clinical AI applications","volume":"4","author":"Kwong","year":"2022","journal-title":"Front Digit Health"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0050","doi-asserted-by":"crossref","first-page":"195","DOI":"10.1186\/s12916-019-1426-2","article-title":"Key challenges for delivering clinical impact with artificial intelligence","volume":"17","author":"Kelly","year":"2019","journal-title":"BMC Med."},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0055","first-page":"1","article-title":"\u2018Generalization\u2014a key challenge for responsible AI in patient-facing clinical applications\u2019, npj Digit","volume":"7","author":"Goetz","year":"2024","journal-title":"Med.,"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0060","first-page":"1","article-title":"\u2018Assessing calibration and bias of a deployed machine learning malnutrition prediction model within a large healthcare system\u2019, npj Digit","volume":"7","author":"Liou","year":"2024","journal-title":"Med.,"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0065","first-page":"1","article-title":"\u2018Clinically informed machine learning elucidates the shape of hospice racial disparities within hospitals\u2019, npj Digit","volume":"6","author":"Khayal","year":"2023","journal-title":"Med.,"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0070","doi-asserted-by":"crossref","first-page":"269","DOI":"10.1055\/s-0043-1768731","article-title":"Security and Privacy in Machine Learning for Health Systems: Strategies and Challenges","volume":"32","author":"de Aguiar","year":"2023","journal-title":"Yearb. Med. Inform."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0075","doi-asserted-by":"crossref","first-page":"52138","DOI":"10.1109\/ACCESS.2018.2870052","article-title":"Peeking Inside the Black-Box: A Survey on Explainable Artificial Intelligence (XAI)","volume":"6","author":"Adadi","year":"2018","journal-title":"IEEE Access"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0080","doi-asserted-by":"crossref","unstructured":"A. Jacovi, A. Marasovi\u0107, T. Miller, and Y. Goldberg, \u2018Formalizing Trust in Artificial Intelligence: Prerequisites, Causes and Goals of Human Trust in AI\u2019, arXiv:2010.07487 [cs], Jan. 2021, Accessed: May 06, 2021. [Online]. Available: http:\/\/arxiv.org\/abs\/2010.07487.","DOI":"10.1145\/3442188.3445923"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0085","doi-asserted-by":"crossref","unstructured":"M. Yin, J. Wortman Vaughan, and H. Wallach, \u2018Understanding the Effect of Accuracy on Trust in Machine Learning Models\u2019, in Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems, in CHI \u201919. New York, NY, USA: Association for Computing Machinery, May 2019, pp. 1\u201312. doi: 10.1145\/3290605.3300509.","DOI":"10.1145\/3290605.3300509"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0090","first-page":"1","article-title":"\u2018Deployment and validation of the CLL treatment infection model adjoined to an EHR system\u2019, npj Digit","volume":"7","author":"Agius","year":"2024","journal-title":"Med.,"},{"issue":"3","key":"10.1016\/j.ijmedinf.2025.105857_b0095","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1109\/TSSC.1970.300339","article-title":"The Nearest Neighbor Classification Rule with a Reject Option","volume":"6","author":"Hellman","year":"1970","journal-title":"IEEE Transactions on Systems Science and Cybernetics"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0100","article-title":"Learning Optimal Conformal Classifiers","volume":"arXiv","author":"Stutz","year":"2022","journal-title":"May 06"},{"issue":"59","key":"10.1016\/j.ijmedinf.2025.105857_b0105","first-page":"1823","article-title":"Classification with a Reject Option using a Hinge Loss","volume":"9","author":"Bartlett","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0110","doi-asserted-by":"crossref","unstructured":"J. Brinkrolf and B. Hammer, \u2018Interpretable machine learning with reject option\u2019, at - Automatisierungstechnik, vol. 66, no. 4, pp. 283\u2013290, Apr. 2018, doi: 10.1515\/auto-2017-0123.","DOI":"10.1515\/auto-2017-0123"},{"issue":"9","key":"10.1016\/j.ijmedinf.2025.105857_b0115","doi-asserted-by":"crossref","first-page":"819","DOI":"10.1016\/0098-1354(92)80035-8","article-title":"A neural network architecture that computes its own reliability","volume":"16","author":"Leonard","year":"1992","journal-title":"Comput. Chem. Eng."},{"issue":"3","key":"10.1016\/j.ijmedinf.2025.105857_b0120","doi-asserted-by":"crossref","first-page":"457","DOI":"10.1007\/s10994-021-05946-3","article-title":"Aleatoric and epistemic uncertainty in machine learning: an introduction to concepts and methods","volume":"110","author":"H\u00fcllermeier","year":"2021","journal-title":"Mach. Learn."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0125","doi-asserted-by":"crossref","unstructured":"B. Kompa, J. Snoek, and A. L. Beam, \u2018Second opinion needed: communicating uncertainty in medical machine learning\u2019, npj Digital Medicine, vol. 4, no. 1, Art. no. 1, Jan. 2021, doi: 10.1038\/s41746-020-00367-3.","DOI":"10.1038\/s41746-020-00367-3"},{"issue":"1","key":"10.1016\/j.ijmedinf.2025.105857_b0130","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1038\/s42256-018-0004-1","article-title":"The need for uncertainty quantification in machine-assisted medical decision making","volume":"1","author":"Begoli","year":"2019","journal-title":"Nat. Mach. Intell."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0135","doi-asserted-by":"crossref","unstructured":"M. Shen, Y. Bu, P. Sattigeri, S. Ghosh, S. Das, and G. Wornell, \u2018Post-hoc Uncertainty Learning Using a Dirichlet Meta-Model\u2019, Proceedings of the AAAI Conference on Artificial Intelligence, vol. 37, no. 8, Art. no. 8, Jun. 2023, doi: 10.1609\/aaai.v37i8.26167.","DOI":"10.1609\/aaai.v37i8.26167"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0140","unstructured":"Y. Ovadia et al., \u2018Can You Trust Your Model\u2019s Uncertainty? Evaluating Predictive Uncertainty Under Dataset Shift\u2019, arXiv:1906.02530 [cs, stat], Dec. 2019, Accessed: Jul. 19, 2021. [Online]. Available: http:\/\/arxiv.org\/abs\/1906.02530."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0145","unstructured":"S. Saria and A. Subbaswamy, \u2018Tutorial: Safe and Reliable Machine Learning\u2019, Apr. 2019, Accessed: Mar. 10, 2020. [Online]. Available: https:\/\/arxiv.org\/abs\/1904.07204v1."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0150","unstructured":"P. Schulam and S. Saria, \u2018Can You Trust This Prediction? Auditing Pointwise Reliability After Learning\u2019, Jan. 2019, Accessed: Mar. 10, 2020. [Online]. Available: https:\/\/arxiv.org\/abs\/1901.00403v2."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0155","doi-asserted-by":"crossref","unstructured":"G. Nicora, M. Rios, A. Abu-Hanna, and R. Bellazzi, \u2018Evaluating Pointwise Reliability of Machine Learning prediction\u2019, Journal of Biomedical Informatics, p. 103996, Gennaio 2022, doi: 10.1016\/j.jbi.2022.103996.","DOI":"10.1016\/j.jbi.2022.103996"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0160","first-page":"925","article-title":"A Reliable Machine Learning Approach applied to Single-Cell Classification in Acute Myeloid Leukemia","volume":"2020","author":"Nicora","year":"2021","journal-title":"AMIA Annu. Symp. Proc."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0165","doi-asserted-by":"crossref","unstructured":"M. T. Ribeiro, S. Singh, and C. Guestrin, \u2018\u201cWhy Should I Trust You?\u201d: Explaining the Predictions of Any Classifier\u2019, arXiv:1602.04938 [cs, stat], Aug. 2016, Accessed: May 03, 2021. [Online]. Available: http:\/\/arxiv.org\/abs\/1602.04938.","DOI":"10.18653\/v1\/N16-3020"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0170","unstructured":"S. Lundberg and S.-I. Lee, \u2018A Unified Approach to Interpreting Model Predictions\u2019, arXiv:1705.07874 [cs, stat], Nov. 2017, Accessed: Nov. 03, 2021. [Online]. Available: http:\/\/arxiv.org\/abs\/1705.07874."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0175","doi-asserted-by":"crossref","DOI":"10.1016\/j.artmed.2022.102471","article-title":"Why did AI get this one wrong? \u2014 Tree-based explanations of machine learning model predictions","volume":"135","author":"Parimbelli","year":"2023","journal-title":"Artif. Intell. Med."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0180","doi-asserted-by":"crossref","first-page":"93","DOI":"10.1016\/j.neucom.2020.04.057","article-title":"An analysis on the use of autoencoders for representation learning: Fundamentals, learning task case studies, explainability and challenges","volume":"404","author":"Charte","year":"2020","journal-title":"Neurocomputing"},{"issue":"44","key":"10.1016\/j.ijmedinf.2025.105857_b0185","doi-asserted-by":"crossref","first-page":"27162","DOI":"10.1073\/pnas.2005013117","article-title":"Overparameterized neural networks implement associative memory","volume":"117","author":"Radhakrishnan","year":"2020","journal-title":"Proc. Natl. Acad. Sci."},{"issue":"2","key":"10.1016\/j.ijmedinf.2025.105857_b0190","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1007\/s10462-023-10662-6","article-title":"Autoencoders and their applications in machine learning: a survey","volume":"57","author":"Berahmand","year":"2024","journal-title":"Artif. Intell. Rev."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0195","series-title":"Advances in Intelligent Data Analysis XVIII","first-page":"444","article-title":"Aleatoric and Epistemic Uncertainty with Random Forests","author":"Shaker","year":"2020"},{"key":"10.1016\/j.ijmedinf.2025.105857_b0200","article-title":"Clinical scoring methods for multiple sclerosis","volume":"vol. 36 Suppl","author":"Jh","year":"1994","journal-title":"Ann. Neurol."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0205","unstructured":"T. M. Buonocore et al., \u2018Predicting and Explaining Risk of Disease Worsening Using Temporal Features in Multiple Sclerosis\u2019, Proceedings of the Working Notes of CLEF 2023 - Conference and Labs of the Evaluation Forum, vol. 3497, pp. 1210\u20131218."},{"key":"10.1016\/j.ijmedinf.2025.105857_b0210","doi-asserted-by":"crossref","unstructured":"K. Zhou, Z. Liu, Y. Qiao, T. Xiang and C. C. Loy, \u201cDomain Generalization: A Survey,\u201d in IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 45, no. 4, pp. 4396-4415, 1 April 2023, doi: 10.1109\/TPAMI.2022.3195549.","DOI":"10.1109\/TPAMI.2022.3195549"}],"container-title":["International Journal of Medical Informatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1386505625000747?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1386505625000747?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,3,8]],"date-time":"2025-03-08T23:18:50Z","timestamp":1741475930000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1386505625000747"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5]]},"references-count":42,"alternative-id":["S1386505625000747"],"URL":"https:\/\/doi.org\/10.1016\/j.ijmedinf.2025.105857","relation":{},"ISSN":["1386-5056"],"issn-type":[{"value":"1386-5056","type":"print"}],"subject":[],"published":{"date-parts":[[2025,5]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"RelAI: an automated approach to judge pointwise ML prediction reliability","name":"articletitle","label":"Article Title"},{"value":"International Journal of Medical Informatics","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.ijmedinf.2025.105857","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 The Author(s). Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"105857"}}