{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T17:05:17Z","timestamp":1784048717658,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,24]],"date-time":"2026-07-24T00:00:00Z","timestamp":1784851200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"N&#x5c;&#x2f;A","award":["N&#x5c;&#x2f;A"],"award-info":[{"award-number":["N&#x5c;&#x2f;A"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,25]]},"DOI":"10.1145\/3805713.3820425","type":"proceedings-article","created":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T16:08:31Z","timestamp":1784045311000},"page":"230-240","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["<scp>DoGMaTiQ:<\/scp>\n                    Automated Generation of Question-and-Answer Nuggets for Report Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5779-1662","authenticated-orcid":false,"given":"Bryan","family":"Li","sequence":"first","affiliation":[{"name":"Google, New York, NY, USA and University of Pennsylvania, Philadelphia, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9931-2861","authenticated-orcid":false,"given":"William","family":"Walden","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2404-7570","authenticated-orcid":false,"given":"Yu","family":"Hou","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0603-1655","authenticated-orcid":false,"given":"Gabrielle Kaili-May","family":"Liu","sequence":"additional","affiliation":[{"name":"Yale University, New Haven, CT, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8196-1943","authenticated-orcid":false,"given":"Chris","family":"Callison-Burch","sequence":"additional","affiliation":[{"name":"University of Pennsylvania, Philadelphia, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1624-3907","authenticated-orcid":false,"given":"Laura","family":"Dietz","sequence":"additional","affiliation":[{"name":"University of New Hampshire, Durham, NH, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,24]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Anthropic. 2024. Introducing Claude 3.5 Sonnet. https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet Accessed: 2026-01-20."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3744382"},{"key":"e_1_3_2_1_3_1","volume-title":"LLM-based relevance assessment still can't replace human relevance assessment. arXiv preprint arXiv:2412.17156","author":"Clarke Charles LA","year":"2024","unstructured":"Charles LA Clarke and Laura Dietz. 2024. LLM-based relevance assessment still can't replace human relevance assessment. arXiv preprint arXiv:2412.17156 (2024)."},{"key":"e_1_3_2_1_4_1","volume-title":"V9","author":"Deutsch Daniel","year":"2021","unstructured":"Daniel Deutsch, Tania Bedrax-Weiss, and Dan Roth. 2021. Towards question-answering as an automatic metric for evaluating the content quality of a summary. Transactions of the Association for Computational Linguistics, V9 (2021), 774-789."},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM","author":"Dietz Laura","year":"2024","unstructured":"Laura Dietz. 2024. A workbench for autograding retrieve\/generate systems. In Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 1963-1972."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3805712.3809981"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 48th European Conference on Information Retrieval (ECIR","author":"Dietz Laura","year":"2026","unstructured":"Laura Dietz, Bryan Li, Gabrielle Liu, Jia-Huei Ju, Eugene Yang, Dawn Lawrie, William Walden, and James Mayfield. 2026 b. Incorporating Q&A Nuggets into Retrieval-Augmented Generation. In Proceedings of the 48th European Conference on Information Retrieval (ECIR 2026). Springer."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 48th European Conference on Information Retrieval (ECIR","author":"Dietz Laura","year":"2026","unstructured":"Laura Dietz, Bryan Li, Eugene Yang, Dawn Lawrie, William Walden, and James Mayfield. 2026 c. Insider Knowledge: How Much Can RAG Systems Gain from Evaluation Secrets?. In Proceedings of the 48th European Conference on Information Retrieval (ECIR 2026). Springer."},{"key":"e_1_3_2_1_9_1","first-page":"3938","volume-title":"2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL HLT","author":"Eyal Matan","year":"2019","unstructured":"Matan Eyal, Tal Baumel, and Michael Elhadad. 2019. Question answering as an automatic evaluation metric for news article summarization. In 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL HLT 2019. Association for Computational Linguistics (ACL), 3938-3948."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of LLM4Eval: The First Workshop on Large Language Models for Evaluation in Information Retrieval.","author":"Farzi Naghmeh","year":"2024","unstructured":"Naghmeh Farzi and Laura Dietz. 2024a. Exam: Llm-based answerability metrics for ir evaluation. In Proceedings of LLM4Eval: The First Workshop on Large Language Models for Evaluation in Information Retrieval."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672511"},{"key":"e_1_3_2_1_12_1","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan et al. 2024. The llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2024\/889"},{"key":"e_1_3_2_1_14_1","volume-title":"Overview of TREC 2024 biomedical generative retrieval (BioGen) track. arXiv preprint arXiv:2411","author":"Gupta Deepak","year":"2024","unstructured":"Deepak Gupta, Dina Demner-Fushman, William Hersh, Steven Bedrick, and Kirk Roberts. 2024. Overview of TREC 2024 biomedical generative retrieval (BioGen) track. arXiv preprint arXiv:2411.18069 (2024)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1224"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2723-2727","author":"Krisztian Balog Weronika","year":"2025","unstructured":"Weronika ?ajewska and Krisztian Balog. 2025. Ginger: Grounded information nugget-based generation of responses. In Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval. 2723-2727."},{"key":"e_1_3_2_1_17_1","volume-title":"Overview of the trec 2024 neuclir track. arXiv preprint arXiv:2509.14355","author":"Lawrie Dawn","year":"2025","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Paul McNamee, Douglas W Oard, Luca Soldaini, and Eugene Yang. 2025a. Overview of the trec 2024 neuclir track. arXiv preprint arXiv:2509.14355 (2025)."},{"key":"e_1_3_2_1_18_1","volume-title":"The Thirty-Fourth Text REtrieval Conference Proceedings (TREC2025)","author":"Lawrie Dawn","year":"2025","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Luca Soldaini, Eugene Yang, and Andrew Yates. 2025b. Overview of the trec 2025 RAGTIME track. The Thirty-Fourth Text REtrieval Conference Proceedings (TREC2025) (2025)."},{"key":"e_1_3_2_1_19_1","first-page":"439","article-title":"PAXQA: Generating Cross-lingual Question Answering Examples at Training Scale","volume":"2023","author":"Li Bryan","year":"2023","unstructured":"Bryan Li and Chris Callison-Burch. 2023. PAXQA: Generating Cross-lingual Question Answering Examples at Training Scale. In Findings of the Association for Computational Linguistics: EMNLP 2023. 439-454.","journal-title":"Findings of the Association for Computational Linguistics: EMNLP"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the Human Language Technology Conference of the NAACL, Main Conference. 383-390","author":"Lin Jimmy","year":"2006","unstructured":"Jimmy Lin and Dina Demner-Fushman. 2006. Will pyramids built of nuggets topple over?. In Proceedings of the Human Language Technology Conference of the NAACL, Main Conference. 383-390."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-naacl.236"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","first-page":"15669","DOI":"10.18653\/v1\/2025.findings-emnlp.849","volume-title":"XRAG: Cross-lingual Retrieval-Augmented Generation. In Findings of the Association for Computational Linguistics: EMNLP 2025","author":"Liu Wei","year":"2025","unstructured":"Wei Liu, Sony Trenous, Leonardo F. R. Ribeiro, Bill Byrne, and Felix Hieber. 2025. XRAG: Cross-lingual Retrieval-Augmented Generation. In Findings of the Association for Computational Linguistics: EMNLP 2025. Association for Computational Linguistics, Suzhou, China, 15669-15690. https:\/\/aclanthology.org\/2025.findings-emnlp.849\/"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3592032"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657846"},{"key":"e_1_3_2_1_25_1","volume-title":"V4, 2","author":"Nenkova Ani","year":"2007","unstructured":"Ani Nenkova, Rebecca Passonneau, and Kathleen McKeown. 2007. The pyramid method: Incorporating human content selection variation in summarization evaluation. ACM Transactions on Speech and Language Processing (TSLP), V4, 2 (2007), 4-es."},{"key":"e_1_3_2_1_26_1","volume-title":"Initial nugget evaluation results for the trec 2024 rag track with the autonuggetizer framework. arXiv preprint arXiv:2411.09607","author":"Pradeep Ronak","year":"2024","unstructured":"Ronak Pradeep, Nandan Thakur, Shivani Upadhyay, Daniel Campos, Nick Craswell, and Jimmy Lin. 2024. Initial nugget evaluation results for the trec 2024 rag track with the autonuggetizer framework. arXiv preprint arXiv:2411.09607 (2024)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3730090"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval. ACM, 1003-1012","author":"Rajput Shahzad K","year":"2011","unstructured":"Shahzad K Rajput, V Pavlu, and Javed A Aslam. 2011. A nugget-based evaluation paradigm to address the scalability and reusability issues of information retrieval test collections. In Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval. ACM, 1003-1012."},{"key":"e_1_3_2_1_29_1","volume-title":"Researchy questions: A dataset of multi-perspective, decompositional questions for llm web agents. arXiv preprint arXiv:2402.17896","author":"Rosset Corby","year":"2024","unstructured":"Corby Rosset, Ho-Lam Chung, Guanghui Qin, Ethan C Chau, Zhuo Feng, Ahmed Awadallah, Jennifer Neville, and Nikhil Rao. 2024. Researchy questions: A dataset of multi-perspective, decompositional questions for llm web agents. arXiv preprint arXiv:2402.17896 (2024)."},{"key":"e_1_3_2_1_30_1","first-page":"136","article-title":"EXAM: How to Evaluate Retrieve-and-Generate Systems for Users Who Do Not (Yet) Know What They Want","author":"Sander David P","year":"2021","unstructured":"David P Sander and Laura Dietz. 2021. EXAM: How to Evaluate Retrieve-and-Generate Systems for Users Who Do Not (Yet) Know What They Want.. In DESIRES. 136-146.","journal-title":"DESIRES."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-8613"},{"key":"e_1_3_2_1_32_1","volume-title":"Hoa Trang Dang, and Jimmy Lin","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Daniel Campos, Nick Craswell, Ian Soboroff, Hoa Trang Dang, and Jimmy Lin. 2024a. A large-scale study of relevance assessments with large language models: An initial look. arXiv preprint arXiv:2411.08275 (2024)."},{"key":"e_1_3_2_1_33_1","volume-title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Nick Craswell, and Jimmy Lin. 2024b. UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519 (2024)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073483.1073520"},{"key":"e_1_3_2_1_35_1","first-page":"54","volume-title":"Overview of the TREC 2003 question answering track.. In Trec","volume":"2003","author":"Voorhees Ellen M","year":"2003","unstructured":"Ellen M Voorhees and Hoa Trang Dang. 2003. Overview of the TREC 2003 question answering track.. In Trec, Vol. V2003. 54-68."},{"key":"e_1_3_2_1_36_1","volume-title":"Yu Hou, et al.","author":"Walden William","year":"2025","unstructured":"William Walden, Marc Mason, Orion Weller, Laura Dietz, John Conroy, Neil Molino, Hannah Recknor, Bryan Li, Gabrielle Kaili-May Liu, Yu Hou, et al., 2025. Auto-argue: Llm-based report generation evaluation. arXiv preprint arXiv:2509.26184 (2025)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56060-6_4"},{"key":"e_1_3_2_1_38_1","volume-title":"RewardAnything: Generalizable Principle-Following Reward Models. arXiv preprint arXiv:2506.03637","author":"Yu Zhuohao","year":"2025","unstructured":"Zhuohao Yu, Jiali Zeng, Weizheng Gu, Yidong Wang, Jindong Wang, Fandong Meng, Jie Zhou, Yue Zhang, Shikun Zhang, and Wei Ye. 2025. RewardAnything: Generalizable Principle-Following Reward Models. arXiv preprint arXiv:2506.03637 (2025)."},{"key":"e_1_3_2_1_39_1","volume-title":"Paraphrase identification with deep learning: A review of datasets and methods","author":"Zhou Chao","year":"2025","unstructured":"Chao Zhou, Cheng Qiu, Lizhen Liang, and Daniel E Acuna. 2025. Paraphrase identification with deep learning: A review of datasets and methods. IEEE Access (2025)."}],"event":{"name":"ICTIR '26: International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2026 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval (ICTIR)"],"original-title":[],"deposited":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T16:12:10Z","timestamp":1784045530000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805713.3820425"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,24]]},"references-count":39,"alternative-id":["10.1145\/3805713.3820425","10.1145\/3805713"],"URL":"https:\/\/doi.org\/10.1145\/3805713.3820425","relation":{},"subject":[],"published":{"date-parts":[[2026,7,24]]},"assertion":[{"value":"2026-07-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}