{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,24]],"date-time":"2026-10-24T01:34:46Z","timestamp":1792805686957,"version":"4.5.0"},"reference-count":69,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,5,29]],"date-time":"2025-05-29T00:00:00Z","timestamp":1748476800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100019779","name":"Qatar National Library","doi-asserted-by":"publisher","award":["HBKU-INT-VPR-TG-02-04"],"award-info":[{"award-number":["HBKU-INT-VPR-TG-02-04"]}],"id":[{"id":"10.13039\/100019779","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Pathology Informatics"],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1016\/j.jpi.2025.100455","type":"journal-article","created":{"date-parts":[[2025,6,5]],"date-time":"2025-06-05T20:24:28Z","timestamp":1749155068000},"page":"100455","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":6,"special_numbering":"C","title":["PathVLM-Eval: Evaluation of open vision language models in histopathology"],"prefix":"10.1016","volume":"18","author":[{"given":"Nauman Ullah","family":"Gilal","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rachida","family":"Zegour","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Khaled","family":"Al-Thelaya","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Erdener","family":"\u00d6zer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marco","family":"Agus","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jens","family":"Schneider","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sabri","family":"Boughorbel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jpi.2025.100455_bb0005","author":"Abdin"},{"key":"10.1016\/j.jpi.2025.100455_bb0010","author":"Achiam"},{"key":"10.1016\/j.jpi.2025.100455_bb0015","first-page":"23716","article-title":"Flamingo: a visual language model for few-shot learning","volume":"35","author":"Alayrac","year":"2022","journal-title":"Adv. Neural Inform. Process. Syst."},{"key":"10.1016\/j.jpi.2025.100455_bb0020","author":"Alexey"},{"key":"10.1016\/j.jpi.2025.100455_bb0025","author":"Bai"},{"key":"10.1016\/j.jpi.2025.100455_bb0030","series-title":"Proceedings of the CLEF 2021 Conference and Labs of the Evaluation Forum-Working Notes, 21-24 September 2021","article-title":"Overview of the vqa-med task at imageclef 2021: visual question answering and generation in the medical domain","author":"Ben Abacha","year":"2021"},{"key":"10.1016\/j.jpi.2025.100455_bb0035","first-page":"1877","article-title":"Language models are few-shot learners. Advances in neural information process-","volume":"33","author":"Brown","year":"2020","journal-title":"ing systems"},{"key":"10.1016\/j.jpi.2025.100455_bb0040","series-title":"European Conference on Computer Vision","first-page":"340","article-title":"Benchlmm: benchmarking cross-style visual capability of large multimodal models","author":"Cai","year":"2025"},{"key":"10.1016\/j.jpi.2025.100455_bb0045","series-title":"European Conference on Computer Vision","first-page":"370","article-title":"Sharegpt4v: improving large multi-modal models with better captions","author":"Chen","year":"2025"},{"key":"10.1016\/j.jpi.2025.100455_bb0050","author":"Chen"},{"key":"10.1016\/j.jpi.2025.100455_bb0055","doi-asserted-by":"crossref","first-page":"220101","DOI":"10.1007\/s11432-024-4231-5","article-title":"How far are we to gpt-4v? closing the gap to commercial multimodal models with open-source suites","volume":"67","author":"Chen","year":"2024","journal-title":"Science China Information Sciences"},{"key":"10.1016\/j.jpi.2025.100455_bb0060","series-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","first-page":"24185","article-title":"Internvl: scaling up vision foundation models and aligning for generic visual-linguistic tasks","author":"Chen","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0065","author":"Chiang"},{"key":"10.1016\/j.jpi.2025.100455_bb0070","author":"Contributors"},{"key":"10.1016\/j.jpi.2025.100455_bb0075","author":"Dai"},{"key":"10.1016\/j.jpi.2025.100455_bb0080","series-title":"Molmo and pixmo: Open weights and open data for state-of-the-art vision-language mod- els, in: Proceedings of the Computer Vision and Pattern Recognition Conference","first-page":"91","author":"Deitke","year":"2025"},{"key":"10.1016\/j.jpi.2025.100455_bb0085","series-title":"Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technolo- gies, volume 1","first-page":"4171","article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"key":"10.1016\/j.jpi.2025.100455_bb0090","author":"Dong"},{"key":"10.1016\/j.jpi.2025.100455_bb0095","series-title":"Proceedings of the 32nd ACM International Conference on Multimedia","first-page":"11198","article-title":"Vlmevalkit: an open-source toolkit for evaluating large multi-modality models","author":"Duan","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0100","author":"Dubey"},{"key":"10.1016\/j.jpi.2025.100455_bb0105","doi-asserted-by":"crossref","first-page":"2111","DOI":"10.3390\/diagnostics11112111","article-title":"Digital pathology workflow implementation at ipatimup","volume":"11","author":"Eloy","year":"2021","journal-title":"Diagnostics"},{"key":"10.1016\/j.jpi.2025.100455_bb0110","doi-asserted-by":"crossref","first-page":"51","DOI":"10.4103\/jpi.jpi_58_17","article-title":"Routine digital pathology workflow: the catania experience","volume":"8","author":"Fraggetta","year":"2017","journal-title":"J. Pathol. Inform."},{"key":"10.1016\/j.jpi.2025.100455_bb0115","first-page":"1","article-title":"Mini-internvl: a flexible-transfer pocket multi- modal model with 5% parameters and 90% performance","volume":"2","author":"Gao","year":"2024","journal-title":"Visual Intel- ligence"},{"key":"10.1016\/j.jpi.2025.100455_bb0120","doi-asserted-by":"crossref","first-page":"10284","DOI":"10.1109\/TPAMI.2024.3437288","article-title":"Unk-vqa: A dataset and a probe into the abstention ability of multi-modal large models","volume":"46","author":"Guo","year":"2024","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelli- gence"},{"key":"10.1016\/j.jpi.2025.100455_bb0125","doi-asserted-by":"crossref","DOI":"10.3389\/frai.2024.1430984","article-title":"Vision-language models for medical report generation and visual question answering: a review","volume":"7","author":"Hartsock","year":"2024","journal-title":"Front. Artif. Intel."},{"key":"10.1016\/j.jpi.2025.100455_bb0130","series-title":"CoRR abs\/2003.10286","article-title":"Pathvqa: 30000+ questions for medical visual question answering","author":"He","year":"2020"},{"key":"10.1016\/j.jpi.2025.100455_bb0135","series-title":"Mini-monkey: multi-scale adaptive cropping for multimodal large language models","author":"Huang","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0140","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/sdata.2018.251","article-title":"A dataset of clinically generated visual questions and answers about radiology images","volume":"5","author":"Lau","year":"2018","journal-title":"Scient. Data"},{"key":"10.1016\/j.jpi.2025.100455_bb0145","series-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","first-page":"13299","article-title":"Seed-bench: benchmarking multimodal large language models","author":"Li","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0150","author":"Li"},{"key":"10.1016\/j.jpi.2025.100455_bb0155","series-title":"International Conference on Machine Learning","first-page":"19730","article-title":"Blip-2: bootstrapping language-image pre-training with frozen image encoders and large language models","author":"Li","year":"2023"},{"key":"10.1016\/j.jpi.2025.100455_bb0160","series-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","first-page":"26763","article-title":"Monkey: image resolution and text label are important things for large multi-modal models","author":"Li","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0165","series-title":"2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI)","first-page":"1650","article-title":"Slake: a semantically-labeled knowledge-enhanced dataset for medical visual question answering","author":"Liu","year":"2021"},{"key":"10.1016\/j.jpi.2025.100455_bb0170","series-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","first-page":"26296","article-title":"Improved baselines with visual instruction tuning","author":"Liu","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0175","author":"Liu"},{"key":"10.1016\/j.jpi.2025.100455_bb0180","series-title":"Llava-next: Improved Reasoning, OCR, and World Knowledge","author":"Liu","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0185","article-title":"Visual instruction tuning","volume":"36","author":"Liu","year":"2024","journal-title":"Adv. Neural Inform. Process. Syst."},{"key":"10.1016\/j.jpi.2025.100455_bb0190","series-title":"European Conference on Computer Vision","first-page":"216","article-title":"Mmbench: is your multi-modal model an all-around player?","author":"Liu","year":"2025"},{"key":"10.1016\/j.jpi.2025.100455_bb0195","doi-asserted-by":"crossref","DOI":"10.1007\/s11432-024-4235-6","article-title":"Ocrbench: on the hidden mystery of ocr in large multimodal models","volume":"67","author":"Liu","year":"2024","journal-title":"Sci. China Inform. Sci."},{"key":"10.1016\/j.jpi.2025.100455_bb0200","author":"Liu"},{"key":"10.1016\/j.jpi.2025.100455_bb0205","author":"Ltd., D"},{"key":"10.1016\/j.jpi.2025.100455_bb0210","author":"Lu"},{"key":"10.1016\/j.jpi.2025.100455_bb0215","author":"Lu"},{"key":"10.1016\/j.jpi.2025.100455_bb0220","series-title":"Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision","first-page":"2200","article-title":"Docvqa: a dataset for vqa on document images","author":"Mathew","year":"2021"},{"key":"10.1016\/j.jpi.2025.100455_bb0225","author":"Nayak"},{"key":"10.1016\/j.jpi.2025.100455_bb0240","series-title":"Gpt-4 technical report","author":"OpenAI","year":"2023"},{"key":"10.1016\/j.jpi.2025.100455_bb0245","series-title":"Findings of the Association for Computational Linguistics","first-page":"1338","article-title":"Dureadervis: a chinese dataset for open-domain document visual question answering","volume":"2022","author":"Qi","year":"2022"},{"key":"10.1016\/j.jpi.2025.100455_bb0250","series-title":"International Conference on Machine Learning","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","author":"Radford","year":"2021"},{"key":"10.1016\/j.jpi.2025.100455_bb0255","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.jpi.2025.100455_bb0260","series-title":"European Conference on Computer Vision","first-page":"56","article-title":"Pathmmu: A massive multimodal expert-level benchmark for understanding and reasoning in pathology","author":"Sun","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0265","author":"Team G"},{"key":"10.1016\/j.jpi.2025.100455_bb0270","author":"Touvron"},{"key":"10.1016\/j.jpi.2025.100455_bb0275","series-title":"Attention is all you need. Advances in neural information processing systems 30","author":"Vaswani","year":"2017"},{"key":"10.1016\/j.jpi.2025.100455_bb0280","author":"Wang"},{"key":"10.1016\/j.jpi.2025.100455_bb0285","author":"Wang"},{"key":"10.1016\/j.jpi.2025.100455_bb0290","author":"Xu"},{"key":"10.1016\/j.jpi.2025.100455_bb0295","doi-asserted-by":"crossref","first-page":"1877","DOI":"10.1109\/TPAMI.2024.3507000","article-title":"Lvlm-ehub: A comprehensive evaluation bench- mark for large vision-language models","volume":"47","author":"Xu","year":"2024","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"10.1016\/j.jpi.2025.100455_bb0300","article-title":"Lamm: language-assisted multi-modal instruction-tuning dataset, framework, and benchmark","volume":"36","author":"Yin","year":"2024","journal-title":"Adv. Neural Inform. Process. Syst."},{"key":"10.1016\/j.jpi.2025.100455_bb0305","author":"Yu"},{"key":"10.1016\/j.jpi.2025.100455_bb0310","series-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","first-page":"9556","article-title":"Mmmu: a massive multi-discipline multimodal understanding and reasoning benchmark for expert agi","author":"Yue","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0315","series-title":"Proceedings of the 28th ACM International Conference on Multimedia","first-page":"2345","article-title":"Medical visual question answering via conditional reasoning","author":"Zhan","year":"2020"},{"key":"10.1016\/j.jpi.2025.100455_bb0320","author":"Zhang"},{"key":"10.1016\/j.jpi.2025.100455_bb0325","author":"Zhang"},{"key":"10.1016\/j.jpi.2025.100455_bb0330","author":"Zhang"},{"key":"10.1016\/j.jpi.2025.100455_bb0335","author":"Zhang"},{"key":"10.1016\/j.jpi.2025.100455_bb0340","author":"Zhang"},{"key":"10.1016\/j.jpi.2025.100455_bb0345","author":"Zhao"},{"key":"10.1016\/j.jpi.2025.100455_bb0350","series-title":"Vision Language Model for Report Generation and Outcome Prediction in CT Pulmonary Angiogram","author":"Zhong","year":"2024"},{"key":"10.1016\/j.jpi.2025.100455_bb0355","author":"Zhu"}],"container-title":["Journal of Pathology Informatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2153353925000409?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2153353925000409?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T11:18:53Z","timestamp":1762600733000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2153353925000409"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8]]},"references-count":69,"alternative-id":["S2153353925000409"],"URL":"https:\/\/doi.org\/10.1016\/j.jpi.2025.100455","relation":{},"ISSN":["2153-3539"],"issn-type":[{"value":"2153-3539","type":"print"}],"subject":[],"published":{"date-parts":[[2025,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"PathVLM-Eval: Evaluation of open vision language models in histopathology","name":"articletitle","label":"Article Title"},{"value":"Journal of Pathology Informatics","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jpi.2025.100455","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 The Author(s). Published by Elsevier Inc. on behalf of Association for Pathology Informatics.","name":"copyright","label":"Copyright"}],"article-number":"100455"}}