{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T10:14:44Z","timestamp":1777889684006,"version":"3.51.4"},"reference-count":49,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,10,19]],"date-time":"2025-10-19T00:00:00Z","timestamp":1760832000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,19]],"date-time":"2025-10-19T00:00:00Z","timestamp":1760832000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,10,19]]},"DOI":"10.1109\/iccv51701.2025.00312","type":"proceedings-article","created":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T19:45:49Z","timestamp":1777491949000},"page":"3258-3267","source":"Crossref","is-referenced-by-count":0,"title":["Trust but Verify: Programmatic VLM Evaluation in the Wild"],"prefix":"10.1109","author":[{"given":"Viraj","family":"Prabhu","sequence":"first","affiliation":[{"name":"Salesforce AI Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Senthil","family":"Purushwalkam","sequence":"additional","affiliation":[{"name":"OpenAI"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"An","family":"Yan","sequence":"additional","affiliation":[{"name":"Salesforce AI Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Caiming","family":"Xiong","sequence":"additional","affiliation":[{"name":"Salesforce AI Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ran","family":"Xu","sequence":"additional","affiliation":[{"name":"Salesforce AI Research"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Semdedup: Data-efficient learning at web-scale through semantic deduplication","author":"Abbas","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref2","article-title":"Phi-3 technical report: A highly capable language model locally on your phone","author":"Abdin","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref3","article-title":"Gpt-4 technical report","author":"Achiam","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref4","article-title":"Palm 2 technical report","author":"Anil","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref5","article-title":"Blip3-kale: Knowledge augmented largescale dense captions","author":"Awadalla","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref6","article-title":"Qwen technical report","author":"Bai","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref7","volume-title":"Executive order on the safe, secure, and trustworthy development and use of artificial intelligence","author":"Biden","year":"2023"},{"key":"ref8","article-title":"On the opportunities and risks of foundation models","author":"Bommasani","year":"2021","journal-title":"arXiv preprint arXiv"},{"key":"ref9","article-title":"Once-for-all: Train one network and specialize it for efficient deployment","author":"Cai","year":"2019","journal-title":"arXiv preprint arXiv"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-024-4231-5"},{"key":"ref11","article-title":"Davidsonian scene graph: Improving reliability in fine-grained evaluation for text-to-image generation","volume-title":"The Twelfth International Conference on Learning Representations","author":"Cho"},{"key":"ref12","article-title":"The llama 3 herd of models","author":"Dubey","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.6"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i16.29771"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01436"},{"key":"ref16","article-title":"Ciem: Contrastive instruction evaluation method for better instruction tuning","volume-title":"NeurIPS 2023 Workshop on Instruction Tuning and Instruction Following","author":"Hu"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01274"},{"key":"ref18","article-title":"The platonic representation hypothesis","author":"Huh","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref19","article-title":"Faithscore: Evaluating hallucinations in large vision-language models","author":"Jing","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref20","article-title":"What if\u2026?: Counterfactual inception to mitigate hallucination effects in large multimodal models","author":"Kim","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.248"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01316"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.20"},{"key":"ref25","article-title":"Hallusionbench: You see what you think? or you think what you see? an image-context reasoning benchmark challenging for gpt-4v (ision), llava-1.5, and other multi-modality models","author":"Liu","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref26","article-title":"Mitigating hallucination in large multi-modal models via robust instruction tuning","volume-title":"Twelfth International Conference on Learning Representations","author":"Liu"},{"key":"ref27","volume-title":"Llava-next: Improved reasoning, ocr, and world knowledge","author":"Liu","year":"2024"},{"key":"ref28","article-title":"Visual instruction tuning","volume":"36","author":"Liu","year":"2024","journal-title":"Advances in neural information processing systems"},{"key":"ref29","article-title":"A survey on hallucination in large vision-language models","author":"Liu","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72970-6_3"},{"key":"ref31","article-title":"Negative object presence evaluation (nope) to measure object hallucination in vision-language models","author":"Lovenia","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref32","article-title":"Wildvision: Evaluating vision-language models in the wild with human preferences","author":"Lu","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref33","article-title":"Mistral","volume-title":"Announcing pixtral 12b","year":"2024"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73027-6_17"},{"key":"ref35","first-page":"9073","article-title":"Synthetic visual genome","volume-title":"Proceedings of the Computer Vision and Pattern Recognition Conference","author":"Sung Park"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58558-7_38"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1437"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.775"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.01092"},{"key":"ref41","article-title":"Gemini: a family of highly capable multimodal models","author":"Team","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref42","article-title":"Llms still can\u2019t plan; can lrms? a preliminary evaluation of openai\u2019s o1 on planbench","author":"Valmeekam","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref43","article-title":"An llmfree multi-dimensional benchmark for mllms hallucination evaluation","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref44","article-title":"Evaluation and analysis of hallucination in large vision-language models","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv"},{"key":"ref45","first-page":"23318","article-title":"Ofa: Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework","volume-title":"International conference on machine learning","author":"Wang"},{"key":"ref46","article-title":"Fiha: Autonomous hallucination evaluation in vision-language models with davidson scene graphs","author":"Yan","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01310"},{"key":"ref48","article-title":"Provision: Programmatically scaling vision-centric instruction data for multimodal language models","author":"Zhang","year":"2024","journal-title":"arXiv preprint arXiv"},{"key":"ref49","article-title":"Segment everything everywhere all at once","volume":"36","author":"Zou","year":"2024","journal-title":"Advances in Neural Information Processing Systems"}],"event":{"name":"2025 IEEE\/CVF International Conference on Computer Vision (ICCV)","location":"Honolulu, HI, USA","start":{"date-parts":[[2025,10,19]]},"end":{"date-parts":[[2025,10,25]]}},"container-title":["2025 IEEE\/CVF International Conference on Computer Vision (ICCV)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11443115\/11443287\/11445499.pdf?arnumber=11445499","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T05:15:08Z","timestamp":1777612508000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11445499\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,19]]},"references-count":49,"URL":"https:\/\/doi.org\/10.1109\/iccv51701.2025.00312","relation":{},"subject":[],"published":{"date-parts":[[2025,10,19]]}}}