{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:03:38Z","timestamp":1784138618352,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"NSF (National Science Foundation)","award":["2402873"],"award-info":[{"award-number":["2402873"]}]},{"name":"Office of Naval Research","award":["N000142412612"],"award-info":[{"award-number":["N000142412612"]}]},{"name":"Dutch Research Council (NWO)","award":["NWA.1389.20.183"],"award-info":[{"award-number":["NWA.1389.20.183"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3808629","type":"proceedings-article","created":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:06:26Z","timestamp":1784135186000},"page":"3365-3373","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Total Recall QA: A Verifiable Evaluation Suite for Deep Research Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-1410-4897","authenticated-orcid":false,"given":"Mahta","family":"Rafiee","sequence":"first","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0393-8662","authenticated-orcid":false,"given":"Heydar","family":"Soudani","sequence":"additional","affiliation":[{"name":"Radboud University, Nijmegen, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4046-3419","authenticated-orcid":false,"given":"Zahra","family":"Abbasiantaeb","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9447-4172","authenticated-orcid":false,"given":"Mohammad","family":"Aliannejadi","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-9986-482X","authenticated-orcid":false,"given":"Faegheh","family":"Hasibi","sequence":"additional","affiliation":[{"name":"Radboud University, Nijmegen, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0800-3340","authenticated-orcid":false,"given":"Hamed","family":"Zamani","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Anthropic. 2025. Claude Sonnet 4.5 System Card. https:\/\/www.anthropic.com\/claude-sonnet-4-5-system-card. Accessed: 2026-02-12."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-naacl.357"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2024.FINDINGS-ACL.137"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2508.06600"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2505.19253"},{"key":"e_1_3_2_1_6_1","volume-title":"Pushing the Frontier of Open Large Language Models. arXiv preprint arXiv:2512.02556","author":"AI.","year":"2025","unstructured":"DeepSeek-AI. 2025. DeepSeek-V3.2: Pushing the Frontier of Open Large Language Models. arXiv preprint arXiv:2512.02556 (2025)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2024.FINDINGS-ACL.716"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2506.11763"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531857"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-88708-6_29"},{"key":"e_1_3_2_1_11_1","volume-title":"Time Travel in LLMs: Tracing Data Contamination in Large Language Models. In The Twelfth International Conference on Learning Representations, ICLR 2024","author":"Golchin Shahriar","year":"2024","unstructured":"Shahriar Golchin and Mihai Surdeanu. 2024. Time Travel in LLMs: Tracing Data Contamination in Large Language Models. In The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024. OpenReview.net. https:\/\/openreview.net\/forum?id=2Rwq6c3tvr"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-321.recall-overview"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2409.04701"},{"key":"e_1_3_2_1_14_1","unstructured":"Nikita Gupta Riju Chatterjee Lukas Haas Connie Tao Andrew Wang Chang Liu Hidekazu Oiwa Elena Gribovskaya Jan Ackermann John Blitzer Sasha Goldshtein and Dipanjan Das. 2026. DeepSearchQA: Bridging the Comprehensiveness Gap for Deep Research Agents. arXiv:2601.20975 [cs.CL] https:\/\/arxiv.org\/abs\/2601.20975"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2512.17776"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2508.13180"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080751"},{"key":"e_1_3_2_1_18_1","unstructured":"Gautier Izacard Mathilde Caron Lucas Hosseini Sebastian Riedel Piotr Bojanowski Armand Joulin and Edouard Grave. 2022. Unsupervised Dense Information Retrieval with Contrastive Learning. Trans. Mach. Learn. Res. (2022)."},{"key":"e_1_3_2_1_19_1","volume-title":"Second Conference on Language Modeling. https:\/\/openreview.net\/forum?id=Rwhi91ideu","author":"Jin Bowen","year":"2025","unstructured":"Bowen Jin, Hansi Zeng, Zhenrui Yue, Jinsung Yoon, Sercan O Arik, Dong Wang, Hamed Zamani, and Jiawei Han. 2025. Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning. In Second Conference on Language Modeling. https:\/\/openreview.net\/forum?id=Rwhi91ideu"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/N18-2075"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2412.05579"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2507.02592"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2508.15804"},{"key":"e_1_3_2_1_24_1","volume-title":"Search-o1: Agentic Search-Enhanced Large Reasoning Models. CoRR","author":"Li Xiaoxi","year":"2025","unstructured":"Xiaoxi Li, Guanting Dong, Jiajie Jin, Yuyao Zhang, Yujia Zhou, Yutao Zhu, Peitian Zhang, and Zhicheng Dou. 2025a. Search-o1: Agentic Search-Enhanced Large Reasoning Models. CoRR (2025)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.784"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-032-02899-0_1"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1018"},{"key":"e_1_3_2_1_28_1","volume-title":"GPT-4o System Card. arXiv preprint arXiv:2410.21276","author":"AI.","year":"2024","unstructured":"OpenAI. 2024. GPT-4o System Card. arXiv preprint arXiv:2410.21276 (2024)."},{"key":"e_1_3_2_1_29_1","volume-title":"OpenAI GPT-5 System Card. arXiv preprint arXiv:2601.03267","author":"AI.","year":"2025","unstructured":"OpenAI. 2025. OpenAI GPT-5 System Card. arXiv preprint arXiv:2601.03267 (2025). GPT-5.2 update: https:\/\/openai.com\/index\/gpt-5-system-card-update-gpt-5-2\/."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2311.12022"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-319.recall-overview"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","unstructured":"Manasi Sharma Chen Bo Calvin Zhang Chaithanya Bandi Clinton Wang Ankit Aich Huy Nghiem Tahseen Rabbani Ye Htet Brian Jang Sumana Basu Aishwarya Balwani Denis Peskoff Marcos Ayestaran Sean M. Hendryx Brad Kenstler and Bing Liu. 2025. ResearchRubrics: A Benchmark of Prompts and Rubrics For Evaluating Deep Research Agents. CoRR Vol. abs\/2511.07685 (2025). arXiv:2511.07685 doi:10.48550\/ARXIV.2511.07685","DOI":"10.48550\/ARXIV.2511.07685"},{"key":"e_1_3_2_1_35_1","unstructured":"Hao Shen Hang Yang and Zhouhong Gu. 2026. ScholarGym: Benchmarking Deep Research Workflows on Academic Literature Retrieval. arXiv:2601.21654 [cs.AI] https:\/\/arxiv.org\/abs\/2601.21654"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673791.3698415"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2406.06519"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.3233\/SW-233471"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2629489"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2212.03533"},{"key":"e_1_3_2_1_42_1","unstructured":"Shaohan Wang Benfeng Xu Licheng Zhang Mingxuan Du Chiwei Zhu Xiaorui Wang Zhendong Mao and Yongdong Zhang. 2026. Wiki Live Challenge: Challenging Deep Research Agents with Expert-Level Wikipedia Articles. arXiv:2602.01590 [cs.CL] https:\/\/arxiv.org\/abs\/2602.01590"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2504.12516"},{"key":"e_1_3_2_1_44_1","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv Chujie Zheng Dayiheng Liu Fan Zhou Fei Huang Feng Hu Hao Ge Haoran Wei Huan Lin Jialong Tang Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jing Zhou Jingren Zhou Junyang Lin Kai Dang Keqin Bao Kexin Yang Le Yu Lianghao Deng Mei Li Mingfeng Xue Mingze Li Pei Zhang Peng Wang Qin Zhu Rui Men Ruize Gao Shixuan Liu Shuang Luo Tianhao Li Tianyi Tang Wenbiao Yin Xingzhang Ren Xinyu Wang Xinyu Zhang Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yinger Zhang Yu Wan Yuqiong Liu Zekun Wang Zeyu Cui Zhenru Zhang Zhipeng Zhou and Zihan Qiu. 2025. Qwen3 Technical Report. arXiv preprint arXiv:2505.09388 (2025)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488560.3498377"},{"key":"e_1_3_2_1_46_1","volume-title":"ReAct: Synergizing Reasoning and Acting in Language Models. In The Eleventh International Conference on Learning Representations, ICLR.","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik R. Narasimhan, and Yuan Cao. 2023. ReAct: Synergizing Reasoning and Acting in Language Models. In The Eleventh International Conference on Learning Representations, ICLR."}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:07:42Z","timestamp":1784135262000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3808629"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":46,"alternative-id":["10.1145\/3805712.3808629","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3808629","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}