{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T15:17:01Z","timestamp":1783091821925,"version":"3.54.6"},"reference-count":46,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/100008746","name":"National Cancer Center","doi-asserted-by":"publisher","award":["1R01CA285737-01A1"],"award-info":[{"award-number":["1R01CA285737-01A1"]}],"id":[{"id":"10.13039\/100008746","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["1R01CA285737-01A1"],"award-info":[{"award-number":["1R01CA285737-01A1"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["clinicalkey.com","clinicalkey.com.au","clinicalkey.es","clinicalkey.fr","clinicalkey.jp","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Artificial Intelligence in Medicine"],"published-print":{"date-parts":[[2026,8]]},"DOI":"10.1016\/j.artmed.2026.103429","type":"journal-article","created":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T06:24:41Z","timestamp":1777875881000},"page":"103429","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"C","title":["EPPCMinerBen: A novel benchmark for evaluating large language models on electronic patient\u2013provider communication via the patient portal"],"prefix":"10.1016","volume":"178","author":[{"given":"Samah Jamal","family":"Fodeh","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1036-9365","authenticated-orcid":false,"given":"Yan","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8519-864X","authenticated-orcid":false,"given":"Linhai","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7409-2871","authenticated-orcid":false,"given":"Srivani","family":"Talakokkul","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7066-4748","authenticated-orcid":false,"given":"Jordan M.","family":"Alpert","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sarah","family":"Schellhorn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"1","key":"10.1016\/j.artmed.2026.103429_b1","article-title":"Cancer statistics, 2022","volume":"72","author":"Siegel","year":"2022","journal-title":"CA: Cancer J Clin"},{"issue":"6","key":"10.1016\/j.artmed.2026.103429_b2","article-title":"Challenges to oral chemotherapy adherence","volume":"44","author":"Thomas","year":"2019","journal-title":"US Pharm"},{"issue":"11","key":"10.1016\/j.artmed.2026.103429_b3","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2021.33877","article-title":"Attitudes and perceptions of multidisciplinary cancer care clinicians toward telehealth and secure messages","volume":"4","author":"Neeman","year":"2021","journal-title":"JAMA Netw Open"},{"key":"10.1016\/j.artmed.2026.103429_b4","doi-asserted-by":"crossref","first-page":"S21","DOI":"10.1097\/MLR.0b013e3182780917","article-title":"Patient-provider secure messaging in VA: variations in adoption and association with urgent care utilization","volume":"51","author":"Shimada","year":"2013","journal-title":"Med Care"},{"issue":"7","key":"10.1016\/j.artmed.2026.103429_b5","first-page":"418","article-title":"Patient access to an electronic health record with secure messaging: impact on primary care utilization","volume":"13","author":"Zhou","year":"2007","journal-title":"Am J Manag Care"},{"issue":"6","key":"10.1016\/j.artmed.2026.103429_b6","doi-asserted-by":"crossref","first-page":"702","DOI":"10.1093\/jamia\/ocx161","article-title":"Dose effect of patient\u2013care team communication via secure portal messaging on glucose and blood pressure control","volume":"25","author":"Price-Haywood","year":"2018","journal-title":"J Am Med Inform Assoc"},{"key":"10.1016\/j.artmed.2026.103429_b7","doi-asserted-by":"crossref","DOI":"10.1016\/j.compbiomed.2022.105345","article-title":"A regularization method to improve adversarial robustness of neural networks for ECG signal classification","volume":"144","author":"Ma","year":"2022","journal-title":"Comput Biol Med"},{"key":"10.1016\/j.artmed.2026.103429_b8","doi-asserted-by":"crossref","DOI":"10.1016\/j.neucom.2023.126524","article-title":"Improving adversarial robustness of deep neural networks via adaptive margin evolution","volume":"551","author":"Ma","year":"2023","journal-title":"Neurocomputing"},{"key":"10.1016\/j.artmed.2026.103429_b9","doi-asserted-by":"crossref","DOI":"10.1016\/j.cmpb.2023.107687","article-title":"Increasing-margin adversarial (IMA) training to improve adversarial robustness of neural networks","volume":"240","author":"Ma","year":"2023","journal-title":"Comput Methods Programs Biomed"},{"key":"10.1016\/j.artmed.2026.103429_b10","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1016\/j.jbi.2017.08.014","article-title":"Classifying patient portal messages using convolutional neural networks","volume":"74","author":"Sulieman","year":"2017","journal-title":"J Biomed Inform"},{"key":"10.1016\/j.artmed.2026.103429_b11","doi-asserted-by":"crossref","first-page":"110","DOI":"10.1016\/j.ijmedinf.2017.06.004","article-title":"A comparison of rule-based and machine learning approaches for classifying patient portal messages","volume":"105","author":"Cronin","year":"2017","journal-title":"Int J Med Inform"},{"issue":"Suppl 20","key":"10.1016\/j.artmed.2026.103429_b12","doi-asserted-by":"crossref","first-page":"507","DOI":"10.1186\/s12859-018-2543-1","article-title":"Bidirectional long short-term memory with CRF for detecting biomedical event trigger in FastText semantic space","volume":"19","author":"Wang","year":"2018","journal-title":"BMC Bioinformatics"},{"key":"10.1016\/j.artmed.2026.103429_b13","doi-asserted-by":"crossref","unstructured":"Zhang N, Chen M, Bi Z, Liang X, Li L, Shang X, Yin K, Tan C, Xu J, Huang F, et al. CBLUE: A Chinese Biomedical Language Understanding Evaluation Benchmark. In: Proceedings of the 60th annual meeting of the association for computational linguistics (volume 1: long papers). 2022, p. 7888\u2013915.","DOI":"10.18653\/v1\/2022.acl-long.544"},{"key":"10.1016\/j.artmed.2026.103429_b14","series-title":"CCF international conference on natural language processing and Chinese computing","first-page":"447","article-title":"Meddg: an entity-centric medical consultation dataset for entity-aware medical dialogue generation","author":"Liu","year":"2022"},{"key":"10.1016\/j.artmed.2026.103429_b15","series-title":"MediTOD: An english dialogue dataset for medical history taking with comprehensive annotations","author":"Saley","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b16","doi-asserted-by":"crossref","unstructured":"Yan G, Pei J, Ren P, Ren Z, Xin X, Liang H, De Rijke M, Chen Z. ReMeDi: Resources for multi-domain, multi-service, medical dialogues. In: Proceedings of the 45th international ACM SIGIR conference on research and development in information retrieval. 2022, p. 3013\u201324.","DOI":"10.1145\/3477495.3531809"},{"key":"10.1016\/j.artmed.2026.103429_b17","series-title":"Machine learning for healthcare conference","first-page":"391","article-title":"MIMIC-SBDH: a dataset for social and behavioral determinants of health","author":"Ahsan","year":"2021"},{"key":"10.1016\/j.artmed.2026.103429_b18","series-title":"Sdoh-nli: a dataset for inferring social determinants of health from clinical notes","author":"Lelkes","year":"2023"},{"issue":"1","key":"10.1016\/j.artmed.2026.103429_b19","first-page":"1","article-title":"Domain-specific language model pretraining for biomedical natural language processing","volume":"3","author":"Gu","year":"2021","journal-title":"ACM Trans Comput Healthc (HEALTH)"},{"issue":"1","key":"10.1016\/j.artmed.2026.103429_b20","doi-asserted-by":"crossref","first-page":"3280","DOI":"10.1038\/s41467-025-56989-2","article-title":"Benchmarking large language models for biomedical natural language processing applications and recommendations","volume":"16","author":"Chen","year":"2025","journal-title":"Nat Commun"},{"issue":"1","key":"10.1016\/j.artmed.2026.103429_b21","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1038\/s41597-019-0055-0","article-title":"BioWordVec, improving biomedical word embeddings with subword information and mesh","volume":"6","author":"Zhang","year":"2019","journal-title":"Sci Data"},{"key":"10.1016\/j.artmed.2026.103429_b22","series-title":"2019 IEEE international conference on healthcare informatics","first-page":"1","article-title":"BioSentVec: creating sentence embeddings for biomedical texts","author":"Chen","year":"2019"},{"issue":"4","key":"10.1016\/j.artmed.2026.103429_b23","doi-asserted-by":"crossref","first-page":"1234","DOI":"10.1093\/bioinformatics\/btz682","article-title":"BioBERT: a pre-trained biomedical language representation model for biomedical text mining","volume":"36","author":"Lee","year":"2020","journal-title":"Bioinformatics"},{"key":"10.1016\/j.artmed.2026.103429_b24","series-title":"Scifive: a text-to-text transformer model for biomedical literature","author":"Phan","year":"2021"},{"issue":"6","key":"10.1016\/j.artmed.2026.103429_b25","doi-asserted-by":"crossref","first-page":"bbac409","DOI":"10.1093\/bib\/bbac409","article-title":"BioGPT: generative pre-trained transformer for biomedical text generation and mining","volume":"23","author":"Luo","year":"2022","journal-title":"Brief Bioinform"},{"key":"10.1016\/j.artmed.2026.103429_b26","doi-asserted-by":"crossref","unstructured":"Yuan H, Yuan Z, Gan R, Zhang J, Xie Y, Yu S. BioBART: Pretraining and Evaluation of A Biomedical Generative Language Model. In: Proceedings of the 21st workshop on biomedical language processing. 2022, p. 97\u2013109.","DOI":"10.18653\/v1\/2022.bionlp-1.9"},{"key":"10.1016\/j.artmed.2026.103429_b27","doi-asserted-by":"crossref","first-page":"27730","DOI":"10.52202\/068431-2011","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Adv Neural Inf Process Syst"},{"issue":"2","key":"10.1016\/j.artmed.2026.103429_b28","doi-asserted-by":"crossref","first-page":"496","DOI":"10.26599\/BDMA.2024.9020090","article-title":"Large language model for medical images: A survey of taxonomy, systematic review, and future trends","volume":"8","author":"Wang","year":"2025","journal-title":"Big Data Min Anal"},{"key":"10.1016\/j.artmed.2026.103429_b29","series-title":"Mixtral of experts","author":"Jiang","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b30","series-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"10.1016\/j.artmed.2026.103429_b31","series-title":"Gpt-4 technical report","author":"Achiam","year":"2023"},{"key":"10.1016\/j.artmed.2026.103429_b32","series-title":"The AI revolution in medicine: GPT-4 and beyond","author":"Lee","year":"2023"},{"key":"10.1016\/j.artmed.2026.103429_b33","series-title":"FinTagging: An LLM-ready benchmark for extracting and structuring financial information","author":"Wang","year":"2025"},{"issue":"6","key":"10.1016\/j.artmed.2026.103429_b34","doi-asserted-by":"crossref","DOI":"10.2196\/37804","article-title":"Conditional probability joint extraction of nested biomedical events: design of a unified extraction framework based on neural networks","volume":"10","author":"Wang","year":"2022","journal-title":"JMIR Med Inform"},{"issue":"4","key":"10.1016\/j.artmed.2026.103429_b35","doi-asserted-by":"crossref","first-page":"243","DOI":"10.1016\/S0738-3991(02)00012-5","article-title":"The roter interaction analysis system (RIAS): utility and flexibility for analysis of medical interactions","volume":"46","author":"Roter","year":"2002","journal-title":"Patient Educ Couns"},{"key":"10.1016\/j.artmed.2026.103429_b36","series-title":"The llama 3 herd of models","author":"Dubey","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b37","series-title":"Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning","author":"Guo","year":"2025"},{"key":"10.1016\/j.artmed.2026.103429_b38","series-title":"Qwen2.5 technical report","author":"Yang","year":"2025"},{"key":"10.1016\/j.artmed.2026.103429_b39","series-title":"QwQ-32b: Embracing the power of reinforcement learning","author":"Team","year":"2025"},{"key":"10.1016\/j.artmed.2026.103429_b40","series-title":"Gemma 2: Improving open language models at a practical size","author":"Team","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b41","series-title":"Gemma: Open models based on gemini research and technology","author":"Team","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b42","series-title":"Qwen2. 5-1m technical report","author":"Yang","year":"2025"},{"key":"10.1016\/j.artmed.2026.103429_b43","series-title":"Qwen2.5: A party of foundation models","author":"Team","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b44","series-title":"OpenBioLLMs: Advancing open-source large language models for healthcare and life sciences","author":"Ankit Pal","year":"2024"},{"key":"10.1016\/j.artmed.2026.103429_b45","series-title":"Integration of large language models and traditional deep learning for social determinants of health prediction","author":"Landes","year":"2025"},{"key":"10.1016\/j.artmed.2026.103429_b46","series-title":"The language model evaluation harness","author":"Gao","year":"2024"}],"container-title":["Artificial Intelligence in Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0933365726000813?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0933365726000813?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T14:20:30Z","timestamp":1783088430000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0933365726000813"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8]]},"references-count":46,"alternative-id":["S0933365726000813"],"URL":"https:\/\/doi.org\/10.1016\/j.artmed.2026.103429","relation":{},"ISSN":["0933-3657"],"issn-type":[{"value":"0933-3657","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"EPPCMinerBen: A novel benchmark for evaluating large language models on electronic patient\u2013provider communication via the patient portal","name":"articletitle","label":"Article Title"},{"value":"Artificial Intelligence in Medicine","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.artmed.2026.103429","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier B.V. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"103429"}}