{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:06:35Z","timestamp":1784138795857,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3809940","type":"proceedings-article","created":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:06:26Z","timestamp":1784135186000},"page":"3681-3687","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["RubricRAG: Towards Interpretable and Reliable LLM Evaluation via Domain Knowledge Retrieval for Rubric Generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-6907-2530","authenticated-orcid":false,"given":"Kaustubh","family":"Dhole","sequence":"first","affiliation":[{"name":"Department of Computer Science, Emory University, Atlanta, Georgia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3148-5448","authenticated-orcid":false,"given":"Eugene","family":"Agichtein","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Emory University, Atlanta, GA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Optimalthinkingbench: Evaluating over and underthinking in llms. arXiv preprint arXiv:2508.13141","author":"Aggarwal Pranjal","year":"2025","unstructured":"Pranjal Aggarwal, Seungone Kim, Jack Lanchantin, Sean Welleck, Jason Weston, Ilia Kulikov, and Swarnadeep Saha. 2025. Optimalthinkingbench: Evaluating over and underthinking in llms. arXiv preprint arXiv:2508.13141 (2025)."},{"key":"e_1_3_2_1_2_1","volume-title":"Preston Bowman, Joaquin Qui nonero-Candela, Foivos Tsimpourlas, Michael Sharman, Meghan Shah, Andrea Vallone, Alex Beutel, et al.","author":"Arora Rahul K","year":"2025","unstructured":"Rahul K Arora, Jason Wei, Rebecca Soskin Hicks, Preston Bowman, Joaquin Qui nonero-Candela, Foivos Tsimpourlas, Michael Sharman, Meghan Shah, Andrea Vallone, Alex Beutel, et al., 2025. Healthbench: Evaluating large language models towards improved human health. arXiv preprint arXiv:2505.08775 (2025)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664646.3664778"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Susan M Brookhart. 2013. How to create and use rubrics for formative assessment and grading. Ascd.","DOI":"10.4135\/9781452218649.n15"},{"key":"e_1_3_2_1_5_1","unstructured":"Kaustubh Dhole and Eugene Agichtein. 2024. Llm judges for retrieval augmented argumentation. (2024)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.starsem-1.32"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-long.293"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-srw.8"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-emnlp.1123"},{"key":"e_1_3_2_1_10_1","volume-title":"Hashimoto","author":"Dubois Yann","year":"2024","unstructured":"Yann Dubois, Xuechen Li, Rohan Taori, Tianyi Zhang, Ishaan Gulrajani, Jimmy Ba, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. 2024. AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback. arXiv:2305.14387 [cs.LG] https:\/\/arxiv.org\/abs\/2305.14387"},{"key":"e_1_3_2_1_11_1","volume-title":"Luis Espinosa Anke, and Steven Schockaert","author":"Es Shahul","year":"2024","unstructured":"Shahul Es, Jithin James, Luis Espinosa Anke, and Steven Schockaert. 2024. RAGAs: Automated Evaluation of Retrieval Augmented Generation. In Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics: System Demonstrations, Nikolaos Aletras and Orphee De Clercq (Eds.). Association for Computational Linguistics, St. Julians, Malta, 150-158. https:\/\/aclanthology.org\/2024.eacl-demo.16"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672511"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672511"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.351"},{"key":"e_1_3_2_1_15_1","unstructured":"Shashwat Goel Rishi Hazra Dulhan Jayalath Timon Willi Parag Jain William F Shen Ilias Leontiadis Francesco Barbieri Yoram Bachrach Jonas Geiping et al. 2025. Training AI Co-Scientists Using Rubric Rewards. arXiv preprint arXiv:2512.23707 (2025)."},{"key":"e_1_3_2_1_16_1","unstructured":"Abinitha Gourabathina Inkit Padhi Manish Nagireddy Subhajit Chaudhury and Prasanna Sattigeri. [n.d.]. Chain-of-Thought Degrades Abstention in Large Language Models Unless Inverted. ([n.d.])."},{"key":"e_1_3_2_1_17_1","unstructured":"Anisha Gunjal Anthony Wang Elaine Lau Vaskar Nath Yunzhong He Bing Liu and Sean Hendryx. 2025. Rubrics as Rewards: Reinforcement Learning Beyond Verifiable Domains. arXiv:2507.17746 [cs.LG] https:\/\/arxiv.org\/abs\/2507.17746"},{"key":"e_1_3_2_1_18_1","volume-title":"RubiCap: Rubric-Guided Reinforcement Learning for Dense Image Captioning. arXiv preprint arXiv:2603.09160","author":"Huang Tzu-Heng","year":"2026","unstructured":"Tzu-Heng Huang, Sirajul Salekin, Javier Movellan, Frederic Sala, and Manjot Bilkhu. 2026. RubiCap: Rubric-Guided Reinforcement Learning for Dense Image Captioning. arXiv preprint arXiv:2603.09160 (2026)."},{"key":"e_1_3_2_1_19_1","unstructured":"Zenan Huang Yihong Zhuang Guoshan Lu Zeyu Qin Haokai Xu Tianyu Zhao Ru Peng Jiaqi Hu Zhanming Shen Xiaomeng Hu et al. 2025. Reinforcement learning with rubric anchors. arXiv preprint arXiv:2508.12790 (2025)."},{"key":"e_1_3_2_1_20_1","volume-title":"The use of scoring rubrics: Reliability, validity and educational consequences. Educational research review","author":"Jonsson Anders","year":"2007","unstructured":"Anders Jonsson and Gunilla Svingby. 2007. The use of scoring rubrics: Reliability, validity and educational consequences. Educational research review, Vol. 2, 2 (2007), 130-144."},{"key":"e_1_3_2_1_21_1","volume-title":"A survey of reinforcement learning from human feedback. Transactions on Machine Learning Research","author":"Kaufmann Timo","year":"2024","unstructured":"Timo Kaufmann, Paul Weng, Viktor Bengs, and Eyke H\u00fcllermeier. 2024. A survey of reinforcement learning from human feedback. Transactions on Machine Learning Research (2024)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.1285"},{"key":"e_1_3_2_1_23_1","volume-title":"The Twelfth International Conference on Learning Representations.","author":"Kim Seungone","year":"2023","unstructured":"Seungone Kim, Jamin Shin, Yejin Cho, Joel Jang, Shayne Longpre, Hwaran Lee, Sangdoo Yun, Seongjin Shin, Sungdong Kim, James Thorne, et al., 2023. Prometheus: Inducing fine-grained evaluation capability in language models. In The Twelfth International Conference on Learning Representations."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-long.303"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 2021 conference on empirical methods in natural language processing: system demonstrations. 175-184","author":"Lhoest Quentin","year":"2021","unstructured":"Quentin Lhoest, Albert Villanova Del Moral, Yacine Jernite, Abhishek Thakur, Patrick Von Platen, Suraj Patil, Julien Chaumond, Mariama Drame, Julien Plu, Lewis Tunstall, et al., 2021. Datasets: A community library for natural language processing. In Proceedings of the 2021 conference on empirical methods in natural language processing: system demonstrations. 175-184."},{"key":"e_1_3_2_1_26_1","volume-title":"RubricHub: A Comprehensive and Highly Discriminative Rubric Dataset via Automated Coarse-to-Fine Generation. arXiv preprint arXiv:2601.08430","author":"Li Sunzhu","year":"2026","unstructured":"Sunzhu Li, Jiale Zhao, Miteto Wei, Huimin Ren, Yang Zhou, Jingwen Yang, Shunyu Liu, Kaike Zhang, and Wei Chen. 2026. RubricHub: A Comprehensive and Highly Discriminative Rubric Dataset via Automated Coarse-to-Fine Generation. arXiv preprint arXiv:2601.08430 (2026)."},{"key":"e_1_3_2_1_27_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74-81. https:\/\/aclanthology.org\/W04-1013\/"},{"key":"e_1_3_2_1_28_1","volume-title":"Mind your step (by step): Chain-of-thought can reduce performance on tasks where thinking makes humans worse. arXiv preprint arXiv:2410.21333","author":"Liu Ryan","year":"2024","unstructured":"Ryan Liu, Jiayi Geng, Addison J Wu, Ilia Sucholutsky, Tania Lombrozo, and Thomas L Griffiths. 2024. Mind your step (by step): Chain-of-thought can reduce performance on tasks where thinking makes humans worse. arXiv preprint arXiv:2410.21333 (2024)."},{"key":"e_1_3_2_1_29_1","volume-title":"Openrubrics: Towards scalable synthetic rubric generation for reward modeling and llm alignment. arXiv preprint arXiv:2510.07743","author":"Liu Tianci","year":"2025","unstructured":"Tianci Liu, Ran Xu, Tony Yu, Ilgee Hong, Carl Yang, Tuo Zhao, and Haoyu Wang. 2025. Openrubrics: Towards scalable synthetic rubric generation for reward modeling and llm alignment. arXiv preprint arXiv:2510.07743 (2025)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-3457"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311-318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311-318."},{"key":"e_1_3_2_1_33_1","volume-title":"Mohamed Shaaban, John Ling, Sean Shi, et al.","author":"Phan Long","year":"2025","unstructured":"Long Phan, Alice Gatti, Ziwen Han, Nathaniel Li, Josephina Hu, Hugh Zhang, Chen Bo Calvin Zhang, Mohamed Shaaban, John Ling, Sean Shi, et al., 2025. Humanity's last exam. arXiv preprint arXiv:2501.14249 (2025)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.20"},{"key":"e_1_3_2_1_36_1","volume-title":"Hamish Ivison, Varsha Kishore, Jingming Zhuo, Xinran Zhao, Molly Park, Samuel G Finlayson, David Sontag, et al.","author":"Shao Rulin","year":"2025","unstructured":"Rulin Shao, Akari Asai, Shannon Zejiang Shen, Hamish Ivison, Varsha Kishore, Jingming Zhuo, Xinran Zhao, Molly Park, Samuel G Finlayson, David Sontag, et al., 2025. Dr tulu: Reinforcement learning with evolving rubrics for deep research. arXiv preprint arXiv:2511.19399 (2025)."},{"key":"e_1_3_2_1_37_1","volume-title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models. arXiv preprint arXiv:2402.03300","author":"Shao Zhihong","year":"2024","unstructured":"Zhihong Shao, Peiyi Wang, Qihao Zhu, Runxin Xu, Junxiao Song, Xiao Bi, Haowei Zhang, Mingchuan Zhang, YK Li, Yang Wu, et al., 2024. Deepseekmath: Pushing the limits of mathematical reasoning in open language models. arXiv preprint arXiv:2402.03300 (2024)."},{"key":"e_1_3_2_1_38_1","volume-title":"Chaithanya Bandi, Clinton Wang, Ankit Aich, Huy Nghiem, Tahseen Rabbani, Ye Htet, Brian Jang, Sumana Basu, et al.","author":"Sharma Manasi","year":"2025","unstructured":"Manasi Sharma, Chen Bo Calvin Zhang, Chaithanya Bandi, Clinton Wang, Ankit Aich, Huy Nghiem, Tahseen Rabbani, Ye Htet, Brian Jang, Sumana Basu, et al., 2025. Researchrubrics: A benchmark of prompts and rubrics for evaluating deep research agents. arXiv preprint arXiv:2511.07685 (2025)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2026.findings-eacl.335"},{"key":"e_1_3_2_1_40_1","unstructured":"Paloma Sodhi Yueheng Li Jessica Landon Eric Wallace and Kai Chen. 2026. Interpreting Black Box Reward Models. OpenAI Alignment Research Blog. https:\/\/alignment.openai.com\/argo\/"},{"key":"e_1_3_2_1_41_1","volume-title":"Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al.","author":"Srivastava Aarohi","year":"2023","unstructured":"Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al., 2023. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. Transactions on machine learning research (2023)."},{"key":"e_1_3_2_1_42_1","volume-title":"Checklists are better than reward models for aligning language models. arXiv preprint arXiv:2507.18624","author":"Viswanathan Vijay","year":"2025","unstructured":"Vijay Viswanathan, Yanchao Sun, Shuang Ma, Xiang Kong, Meng Cao, Graham Neubig, and Tongshuang Wu. 2025. Checklists are better than reward models for aligning language models. arXiv preprint arXiv:2507.18624 (2025)."},{"key":"e_1_3_2_1_43_1","volume-title":"TRL: Transformers Reinforcement Learning. https:\/\/github.com\/huggingface\/trl","author":"von Werra Leandro","year":"2020","unstructured":"Leandro von Werra, Younes Belkada, Lewis Tunstall, Edward Beeching, Tristan Thrush, Nathan Lambert, Shengyi Huang, Kashif Rasul, and Quentin Gallou\u00e9dec. 2020. TRL: Transformers Reinforcement Learning. https:\/\/github.com\/huggingface\/trl"},{"key":"e_1_3_2_1_44_1","volume-title":"The Thirteenth International Conference on Learning Representations.","author":"Wei Tianjun","unstructured":"Tianjun Wei, Wei Wen, Ruizhi Qiao, Xing Sun, and Jianghong Ma. [n.d.]. RocketEval: Efficient automated LLM evaluation via grading checklist. In The Thirteenth International Conference on Learning Representations."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_1_46_1","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv et al. 2025. Qwen3 technical report. arXiv preprint arXiv:2505.09388 (2025)."},{"key":"e_1_3_2_1_47_1","volume-title":"Proceedings of the 31st International Conference on Computational Linguistics, Owen Rambow, Leo Wanner, Marianna Apidianaki, Hend Al-Khalifa","author":"Ye Junjie","year":"2025","unstructured":"Junjie Ye, Guanyu Li, SongYang Gao, Caishuang Huang, Yilong Wu, Sixian Li, Xiaoran Fan, Shihan Dou, Tao Ji, Qi Zhang, Tao Gui, and Xuanjing Huang. 2025. ToolEyes: Fine-Grained Evaluation for Tool Learning Capabilities of Large Language Models in Real-world Scenarios. In Proceedings of the 31st International Conference on Computational Linguistics, Owen Rambow, Leo Wanner, Marianna Apidianaki, Hend Al-Khalifa, Barbara Di Eugenio, and Steven Schockaert (Eds.). Association for Computational Linguistics, Abu Dhabi, UAE, 156-187. https:\/\/aclanthology.org\/2025.coling-main.12\/"},{"key":"e_1_3_2_1_48_1","volume-title":"ICLR 2024 Workshop on Large Language Model (LLM) Agents.","author":"Ye Seonghyeon","unstructured":"Seonghyeon Ye, Doyoung Kim, Sungdong Kim, Hyeonbin Hwang, Seungone Kim, Yongrae Jo, James Thorne, Juho Kim, and Minjoon Seo. [n.d.]. FLASK: Fine-grained Language Model Evaluation based on Alignment Skill Sets. In ICLR 2024 Workshop on Large Language Model (LLM) Agents."},{"key":"e_1_3_2_1_49_1","unstructured":"Qiyuan Zhang Junyi Zhou Yufei Wang Fuyuan Lyu Yidong Ming Can Xu Qingfeng Sun Kai Zheng Peng Kang Xue Liu et al. 2026. RubricBench: Aligning Model-Generated Rubrics with Human Standards. arXiv preprint arXiv:2603.01562 (2026)."},{"key":"e_1_3_2_1_50_1","volume-title":"Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models. arXiv preprint arXiv:2506.05176","author":"Zhang Yanzhao","year":"2025","unstructured":"Yanzhao Zhang, Mingxin Li, Dingkun Long, Xin Zhang, Huan Lin, Baosong Yang, Pengjun Xie, An Yang, Dayiheng Liu, Junyang Lin, Fei Huang, and Jingren Zhou. 2025. Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models. arXiv preprint arXiv:2506.05176 (2025)."}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:16:38Z","timestamp":1784135798000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3809940"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":50,"alternative-id":["10.1145\/3805712.3809940","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3809940","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}