{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,2]],"date-time":"2026-09-02T18:19:16Z","timestamp":1788373156460,"version":"build-2803163510"},"reference-count":20,"publisher":"Association for Computing Machinery (ACM)","issue":"1","license":[{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/publication-rights-and-licensing-policy"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["SIGIR Forum"],"published-print":{"date-parts":[[2026,6,1]]},"abstract":"<jats:p>\n                    We report on the outcomes of the first BREV-RAG (Beyond Relevance-based EValuation of RAG systems) workshop, a half-day post-conference workshop of ACM SIGIR-AP 2025. The workshop featured five paper presentations and breakout group discussions. After the paper presentations, three evaluation axes\/themes were chosen by the workshop participants and accordingly three discussion groups were formed, namely,\n                    <jats:italic toggle=\"yes\">modesty, diversity and fairness<\/jats:italic>\n                    , and\n                    <jats:italic toggle=\"yes\">specialised domains<\/jats:italic>\n                    groups. We report on what each of the groups discussed during and after the workshop.\n                  <\/jats:p>\n                  <jats:p>\n                    <jats:bold>Date:<\/jats:bold>\n                    10 December 2025.\n                  <\/jats:p>\n                  <jats:p>\n                    <jats:bold>Website:<\/jats:bold>\n                    http:\/\/sakailab.com\/brev-rag\/.\n                  <\/jats:p>","DOI":"10.1145\/3841298.3841301","type":"journal-article","created":{"date-parts":[[2026,9,2]],"date-time":"2026-09-02T18:00:28Z","timestamp":1788372028000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Report on the 1st Workshop on Beyond Relevance-Based EValuation of RAG Systems (BREV-RAG 2025) at SIGIR-AP 2025"],"prefix":"10.1145","volume":"60","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[{"name":"Waseda University\/Naver Corporation, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hanpei","family":"Fang","sequence":"additional","affiliation":[{"name":"Waseda University, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Makoto P.","family":"Kato","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Keping","family":"Bi","sequence":"additional","affiliation":[{"name":"Chinese Academy of Sciences, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hideo","family":"Joho","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Charles L. A.","family":"Clarke","sequence":"additional","affiliation":[{"name":"University of Waterloo, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sijie","family":"Tao","sequence":"additional","affiliation":[{"name":"Waseda University, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Junjie","family":"Wang","sequence":"additional","affiliation":[{"name":"Tsinghua University, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haoxiang","family":"Shi","sequence":"additional","affiliation":[{"name":"Inner Mongolia University of Technology, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Risa","family":"Hiramatsu","sequence":"additional","affiliation":[{"name":"Waseda University, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hengran","family":"Zhang","sequence":"additional","affiliation":[{"name":"Chinese Academy of Sciences, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiaxin","family":"Mao","sequence":"additional","affiliation":[{"name":"Renmin University of China, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuya","family":"Ishihara","sequence":"additional","affiliation":[{"name":"Hitotsubashi University, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Taichi","family":"Motegi","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nuo","family":"Chen","sequence":"additional","affiliation":[{"name":"The Hong Kong Polytechnic University, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhicheng","family":"Dou","sequence":"additional","affiliation":[{"name":"Renmin University of China, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Atsushi","family":"Keyaki","sequence":"additional","affiliation":[{"name":"Hitotsubashi University, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,9,2]]},"reference":[{"key":"e_1_2_1_1_1","volume-title":"A general language assistant as a laboratory for alignment","author":"Askell Amanda","year":"2021","unstructured":"Amanda Askell, Yuntao Bai, Anna Chen, Dawn Drain, Deep Ganguli, Tom Henighan, Andy Jones, Nicholas Joseph, Ben Mann, Nova DasSarma, Nelson Elhage, Zac Hatfield-Dodds, Danny Hernandez, Jackson Kernion, Kamal Ndousse, Catherine Olsson, Dario Amodei, Tom Brown, Jack Clark, Sam McCandlish, Chris Olah, and Jared Kaplan. A general language assistant as a laboratory for alignment, 2021. URL https:\/\/arxiv.org\/abs\/2112.00861."},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673791.3698420"},{"key":"e_1_2_1_3_1","volume-title":"Voorhees. TREC 2013 web track overview. In Proceedings of The Twenty-Second Text REtrieval Conference, TREC 2013","author":"Collins-Thompson Kevyn","year":"2013","unstructured":"Kevyn Collins-Thompson, Paul N. Bennett, Fernando Diaz, Charlie Clarke, and Ellen M. Voorhees. TREC 2013 web track overview. In Proceedings of The Twenty-Second Text REtrieval Conference, TREC 2013, 2013."},{"key":"e_1_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3767695.3769493"},{"key":"e_1_2_1_5_1","first-page":"33","volume-title":"Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025","author":"Hiramatsu Risa","year":"2025","unstructured":"Risa Hiramatsu, Rikiya Takehi, and Tetsuya Sakai. Aspect-based evaluation of personalization and diversification in conversational search systems. In Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025, pages 22\u201333, 2025. URL https:\/\/brev-rag-workshop.github.io\/paper-03.pdf."},{"key":"e_1_2_1_6_1","first-page":"65","volume-title":"Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025","author":"Ishihara Yuya","year":"2025","unstructured":"Yuya Ishihara, Atsushi Keyaki, Hiroaki Yamada, Ryutaro Ohara, and Mihoko Sumida. RAG system for supporting japanese litigation procedures: Faithful response generation complying with legal norms. In Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025, pages 59\u201365, 2025. URL https:\/\/brev-rag-workshop.github.io\/paper-06.pdf."},{"key":"e_1_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00638"},{"key":"e_1_2_1_8_1","volume-title":"Muhammad Faaiz Taufiq, and Hang Li. Trustworthy llms: a survey and guideline for evaluating large language models' alignment","author":"Liu Yang","year":"2024","unstructured":"Yang Liu, Yuanshun Yao, Jean-Francois Ton, Xiaoying Zhang, Ruocheng Guo, Hao Cheng, Yegor Klochkov, Muhammad Faaiz Taufiq, and Hang Li. Trustworthy llms: a survey and guideline for evaluating large language models' alignment, 2024b. URL https:\/\/arxiv.org\/abs\/2308.05374."},{"key":"e_1_2_1_9_1","volume-title":"Faitheval: Can your language model stay faithful to context, even if \"the moon is made of marshmallows","author":"Ming Yifei","year":"2025","unstructured":"Yifei Ming, Senthil Purushwalkam, Shrey Pandit, Zixuan Ke, Xuan-Phi Nguyen, Caiming Xiong, and Shafiq Joty. Faitheval: Can your language model stay faithful to context, even if \"the moon is made of marshmallows\", 2025. URL https:\/\/arxiv.org\/abs\/2410.03727."},{"key":"e_1_2_1_10_1","first-page":"46","volume-title":"Proceedings of the International Workshop on Beyond Relevance-based Evaluation of RAG Systems (BREV-RAG) 2025","author":"Motegi Taichi","year":"2025","unstructured":"Taichi Motegi, Makoto P. Kato, Kazuhisa Hatakeyama, and Yosuke Yurikusa. Retrieval-augmented relevance judgment for specialized domains. In Proceedings of the International Workshop on Beyond Relevance-based Evaluation of RAG Systems (BREV-RAG) 2025, pages 34\u201346, 2025. URL https:\/\/brev-rag-workshop.github.io\/paper-04.pdf."},{"key":"e_1_2_1_11_1","volume-title":"Graded relevance assessments and graded relevance measures of NTCIR: A survey of the first twenty years. CoRR, abs\/1903.11272","author":"Sakai Tetsuya","year":"2019","unstructured":"Tetsuya Sakai. Graded relevance assessments and graded relevance measures of NTCIR: A survey of the first twenty years. CoRR, abs\/1903.11272, 2019. URL http:\/\/arxiv.org\/abs\/1903-11272."},{"key":"e_1_2_1_12_1","volume-title":"Joint Proceedings of the SIGIR-AP 2024 Workshops EMTCIR 2024 and UM-CIR 2024","author":"Sakai Tetsuya","year":"2024","unstructured":"Tetsuya Sakai. Evaluating system responses based on overconfidence and underconfidence. In Joint Proceedings of the SIGIR-AP 2024 Workshops EMTCIR 2024 and UM-CIR 2024, 2024. URL https:\/\/ceur-ws.org\/Vol-3854\/emtcir-1.pdf."},{"key":"e_1_2_1_13_1","volume-title":"NTCIR","author":"Sakai Tetsuya","year":"2013","unstructured":"Tetsuya Sakai, Zhicheng Dou, Takehiro Yamamoto, Yiqun Liu, Min Zhang, Ruihua Song, MP Kato, and M Iwata. Overview of the ntcir-10 intent-2 task. In NTCIR, 2013."},{"key":"e_1_2_1_14_1","first-page":"21","volume-title":"Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025","author":"Sakai Tetsuya","year":"2025","unstructured":"Tetsuya Sakai, Sijie Tao, Atsuya Ishikawa, Hanpei Fang, Ziliang Zhao, Yujia Zhou, Nuo Chen, and Young-In Song. Evaluating the modesty of RAG systems at the NTCIR-19 R2C2 task: Potential challenges. In Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025, pages 5\u201321, 2025. URL https:\/\/brev-rag-workshop.github.io\/paper-02.pdf."},{"key":"e_1_2_1_15_1","volume-title":"NTCIR","author":"Song Ruihua","year":"2011","unstructured":"Ruihua Song, Min Zhang, Tetsuya Sakai, Makoto P Kato, Yiqun Liu, Miho Sugimoto, Qinglei Wang, and Naoki Orii. Overview of the ntcir-9 intent task. In NTCIR, 2011."},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-024-00976-7"},{"key":"e_1_2_1_17_1","first-page":"58","volume-title":"Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025","author":"Tao Sijie","year":"2025","unstructured":"Sijie Tao and Tetsuya Sakai. Measuring group fairness differences in RAG pipelines: A pilot study. In Proceedings of the International Workshop on Beyond Relevance-based EValuation of RAG Systems (BREV-RAG) 2025, pages 47\u201358, 2025. URL https:\/\/brev-rag-workshop.github.io\/paper-05.pdf."},{"key":"e_1_2_1_18_1","volume-title":"Overview of the ntcir-17 fairweb-1 task","author":"Tao Sijie","year":"2023","unstructured":"Sijie Tao, Nuo Chen, Tetsuya Sakai, Zhumin Chu, Hiromi Arai, Ian Soboroff, Nicola Ferro, and Maria Maistro. Overview of the ntcir-17 fairweb-1 task. 2023."},{"key":"e_1_2_1_19_1","volume-title":"Overview of the ntcir-18 fairweb-2 task","author":"Tao Sijie","year":"2025","unstructured":"Sijie Tao, Tetsuya Sakai, Junjie Wang, Hanpei Fang, Yuxiang Zhang, Haitao Li, Yiteng Tu, Nuo Chen, and Maria Maistro. Overview of the ntcir-18 fairweb-2 task. 2025."},{"key":"e_1_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484053"}],"container-title":["ACM SIGIR Forum"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3841298.3841301","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,2]],"date-time":"2026-09-02T18:00:36Z","timestamp":1788372036000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3841298.3841301"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,1]]},"references-count":20,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2026,6,1]]}},"alternative-id":["10.1145\/3841298.3841301"],"URL":"https:\/\/doi.org\/10.1145\/3841298.3841301","relation":{},"ISSN":["0163-5840"],"issn-type":[{"value":"0163-5840","type":"print"}],"subject":[],"published":{"date-parts":[[2026,6,1]]},"assertion":[{"value":"2026-09-02","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}