{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,5]],"date-time":"2026-09-05T03:03:13Z","timestamp":1788577393694,"version":"build-2803163510"},"reference-count":24,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,8,29]],"date-time":"2025-08-29T00:00:00Z","timestamp":1756425600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"NIH","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Biomedical Informatics"],"published-print":{"date-parts":[[2025,10]]},"DOI":"10.1016\/j.jbi.2025.104900","type":"journal-article","created":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T15:15:30Z","timestamp":1755962130000},"page":"104900","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":4,"special_numbering":"C","title":["CeRTS: certainty retrieval token search in large language model clinical information extraction"],"prefix":"10.1016","volume":"170","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-0585-2062","authenticated-orcid":false,"given":"Lars E.","family":"Schimmelpfennig","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kriti","family":"Bhattarai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0780-8710","authenticated-orcid":false,"given":"Inez Y.","family":"Oh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jake","family":"Lever","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Obi L.","family":"Griffith","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6388-446X","authenticated-orcid":false,"given":"Malachi","family":"Griffith","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Albert M.","family":"Lai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5219-9996","authenticated-orcid":false,"given":"Zachary B.","family":"Abrams","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"1","key":"10.1016\/j.jbi.2025.104900_b0005","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41746-023-00957-x","article-title":"Zero-shot interpretable phenotyping of postpartum hemorrhage using large language models","volume":"6","author":"Alsentzer","year":"2023","journal-title":"Npj Digit Med."},{"key":"10.1016\/j.jbi.2025.104900_b0010","doi-asserted-by":"crossref","unstructured":"McInerney DJ, Young G, Meent JW van de, Wallace BC. CHiLL: Zero-shot Custom Interpretable Feature Extraction from Clinical Notes with Large Language Models. Published online October 19, 2023. doi:10.48550\/arXiv.2302.12343.","DOI":"10.18653\/v1\/2023.findings-emnlp.568"},{"issue":"1","key":"10.1016\/j.jbi.2025.104900_b0015","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1109\/TCBB.2018.2849968","article-title":"Natural language processing for ehr-based computational phenotyping","volume":"16","author":"Zeng","year":"2019","journal-title":"IEEE\/ACM Trans. Comput. Biol. Bioinform."},{"issue":"3","key":"10.1016\/j.jbi.2025.104900_b0020","doi-asserted-by":"crossref","DOI":"10.1093\/jamiaopen\/ooae060","article-title":"Leveraging GPT-4 for identifying cancer phenotypes in electronic health records: a performance comparison between GPT-4, GPT-3.5-turbo, Flan-T5, Llama-3-8B, and spaCy\u2019s rule-based and machine learning-based methods","volume":"7","author":"Bhattarai","year":"2024","journal-title":"JAMIA Open."},{"issue":"12","key":"10.1016\/j.jbi.2025.104900_b0025","doi-asserted-by":"crossref","first-page":"3426","DOI":"10.1038\/s41596-019-0227-6","article-title":"High-throughput phenotyping with electronic medical record data using a common semi-supervised approach (PheCAP)","volume":"14","author":"Zhang","year":"2019","journal-title":"Nat. Protoc."},{"issue":"13","key":"10.1016\/j.jbi.2025.104900_b0030","doi-asserted-by":"crossref","first-page":"1884","DOI":"10.1093\/bioinformatics\/btab019","article-title":"PhenoTagger: a hybrid method for phenotype concept recognition using human phenotype ontology","volume":"37","author":"Luo","year":"2021","journal-title":"Bioinformatics"},{"key":"10.1016\/j.jbi.2025.104900_b0035","doi-asserted-by":"crossref","unstructured":"Rivera M, Godbout JF, Rabbany R, Pelrine K. Combining Confidence Elicitation and Sample-based Methods for Uncertainty Quantification in Misinformation Mitigation. Published online January 30, 2024. doi:10.48550\/arXiv.2401.08694.","DOI":"10.18653\/v1\/2024.uncertainlp-1.12"},{"key":"10.1016\/j.jbi.2025.104900_b0040","unstructured":"Chen J, Mueller J. Quantifying Uncertainty in Answers from any Language Model and Enhancing their Trustworthiness. Published online October 4, 2023. doi:10.48550\/arXiv.2308.16175."},{"key":"10.1016\/j.jbi.2025.104900_b0045","doi-asserted-by":"crossref","unstructured":"Manakul P, Liusie A, Gales MJF. SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models. Published online October 11, 2023. doi:10.48550\/arXiv.2303.08896.","DOI":"10.18653\/v1\/2023.emnlp-main.557"},{"key":"10.1016\/j.jbi.2025.104900_b0050","doi-asserted-by":"crossref","unstructured":"Mahaut M, Aina L, Czarnowska P, Hardalov M, M\u00fcller T, M\u00e0rquez L. Factual Confidence of LLMs: on Reliability and Robustness of Current Estimators. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). ; 2024:4554-4570. doi:10.18653\/v1\/2024.acl-long.250.","DOI":"10.18653\/v1\/2024.acl-long.250"},{"issue":"1","key":"10.1016\/j.jbi.2025.104900_b0055","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1093\/jamia\/ocae254","article-title":"Large language model uncertainty proxies: discrimination and calibration for medical diagnosis and treatment","volume":"32","author":"Savage","year":"2025","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.jbi.2025.104900_b0060","first-page":"1","article-title":"Look before you leap: an exploratory study of uncertainty measurement for large language models","author":"Huang","year":"2025","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jbi.2025.104900_b0065","unstructured":"Ma H, Chen J, Wang G, Zhang C. Estimating LLM Uncertainty with Logits. Published online February 11, 2025. doi:10.48550\/arXiv.2502.00290."},{"key":"10.1016\/j.jbi.2025.104900_b0070","doi-asserted-by":"crossref","unstructured":"Zawistowski K. Unused information in token probability distribution of generative LLM: improving LLM reading comprehension through calculation of expected values. Published online September 26, 2024. doi:10.48550\/arXiv.2406.10267.","DOI":"10.15439\/2024F7639"},{"issue":"2","key":"10.1016\/j.jbi.2025.104900_b0075","doi-asserted-by":"crossref","first-page":"377","DOI":"10.1016\/j.jbi.2008.08.010","article-title":"Research electronic data capture (REDCap)\u2014A metadata-driven methodology and workflow process for providing translational research informatics support","volume":"42","author":"Harris","year":"2009","journal-title":"J. Biomed. Inform."},{"key":"10.1016\/j.jbi.2025.104900_b0080","unstructured":"DeepSeek-AI, Guo D, Yang D, et al. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. Published online January 22, 2025. doi:10.48550\/arXiv.2501.12948."},{"key":"10.1016\/j.jbi.2025.104900_b0085","unstructured":"Team G, Riviere M, Pathak S, et al. Gemma 2: Improving Open Language Models at a Practical Size. Published online October 2, 2024. doi:10.48550\/arXiv.2408.00118."},{"key":"10.1016\/j.jbi.2025.104900_b0090","unstructured":"Teknium R, Quesnelle J, Guang C. Hermes 3 Technical Report. Published online August 15, 2024. doi:10.48550\/arXiv.2408.11857."},{"key":"10.1016\/j.jbi.2025.104900_b0095","doi-asserted-by":"crossref","unstructured":"Mistral Small 3 | Mistral AI. Accessed March 31, 2025. https:\/\/mistral.ai\/news\/mistral-small-3.","DOI":"10.5840\/mr2025312"},{"key":"10.1016\/j.jbi.2025.104900_b0100","unstructured":"Jiang AQ, Sablayrolles A, Roux A, et al. Mixtral of Experts. Published online January 8, 2024. doi:10.48550\/arXiv.2401.04088."},{"key":"10.1016\/j.jbi.2025.104900_b0105","unstructured":"Abdin M, Aneja J, Behl H, et al. Phi-4 Technical Report. Published online December 12, 2024. doi:10.48550\/arXiv.2412.08905."},{"key":"10.1016\/j.jbi.2025.104900_b0110","unstructured":"Yang A, Yu B, Li C, et al. Qwen2.5-1M Technical Report. Published online January 26, 2025. doi:10.48550\/arXiv.2501.15383."},{"key":"10.1016\/j.jbi.2025.104900_b0115","unstructured":"Grattafiori A, Dubey A, Jauhri A, et al. The Llama 3 Herd of Models. Published online November 23, 2024. doi:10.48550\/arXiv.2407.21783."},{"key":"10.1016\/j.jbi.2025.104900_b0120","doi-asserted-by":"crossref","unstructured":"Brier GW. VERIFICATION OF FORECASTS EXPRESSED IN TERMS OF PROBABILITY. Published online January 1, 1950. Accessed March 31, 2025. https:\/\/journals.ametsoc.org\/view\/journals\/mwre\/78\/1\/1520-0493_1950_078_0001_vofeit_{20}_co_2.xml.","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2"}],"container-title":["Journal of Biomedical Informatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1532046425001297?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1532046425001297?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,31]],"date-time":"2026-05-31T08:00:35Z","timestamp":1780214435000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1532046425001297"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10]]},"references-count":24,"alternative-id":["S1532046425001297"],"URL":"https:\/\/doi.org\/10.1016\/j.jbi.2025.104900","relation":{},"ISSN":["1532-0464"],"issn-type":[{"value":"1532-0464","type":"print"}],"subject":[],"published":{"date-parts":[[2025,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"CeRTS: certainty retrieval token search in large language model clinical information extraction","name":"articletitle","label":"Article Title"},{"value":"Journal of Biomedical Informatics","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jbi.2025.104900","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 The Authors. Published by Elsevier Inc.","name":"copyright","label":"Copyright"}],"article-number":"104900"}}