{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:17:35Z","timestamp":1784179055802,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":94,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657846","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"1904-1915","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":28,"title":["On the Evaluation of Machine-Generated Reports"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"first","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8914-2659","authenticated-orcid":false,"given":"Sean","family":"MacAvaney","sequence":"additional","affiliation":[{"name":"University of Glasgow, Glasgow, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0548-5751","authenticated-orcid":false,"given":"Paul","family":"McNamee","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1696-0407","authenticated-orcid":false,"given":"Douglas W.","family":"Oard","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6998-9863","authenticated-orcid":false,"given":"Luca","family":"Soldaini","sequence":"additional","affiliation":[{"name":"Allen Institute for AI, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2363-3014","authenticated-orcid":false,"given":"Ian","family":"Soboroff","sequence":"additional","affiliation":[{"name":"NIST, Gaithersburg, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4148-5430","authenticated-orcid":false,"given":"Orion","family":"Weller","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2357-9807","authenticated-orcid":false,"given":"Efsun","family":"Kayi","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3125-2975","authenticated-orcid":false,"given":"Kate","family":"Sanders","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1784-6526","authenticated-orcid":false,"given":"Marc","family":"Mason","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7967-4109","authenticated-orcid":false,"given":"Noah","family":"Hibbler","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al.","author":"Achiam Josh","year":"2023","unstructured":"Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et al. 2023. GPT-4 technical report. arXiv preprint arXiv:2303.08774 (2023)."},{"key":"e_1_3_2_1_2_1","volume-title":"RetrievalSum: A retrieval enhanced framework for abstractive summarization. arXiv preprint arXiv:2109.07943","author":"An Chenxin","year":"2021","unstructured":"Chenxin An, Ming Zhong, Zhichao Geng, Jianqiang Yang, and Xipeng Qiu. 2021. RetrievalSum: A retrieval enhanced framework for abstractive summarization. arXiv preprint arXiv:2109.07943 (2021)."},{"key":"e_1_3_2_1_3_1","volume-title":"Gabriel Ilharco, Mitchell Wortsman, and Ludwig Schmidt.","author":"Awadalla Anas","year":"2023","unstructured":"Anas Awadalla, Irena Gao, Josh Gardner, Jack Hessel, Yusuf Hanafy, Wanrong Zhu, Kalyani Marathe, Yonatan Bitton, Samir Gadre, Shiori Sagawa, Jenia Jitsev, Simon Kornblith, Pang Wei Koh, Gabriel Ilharco, Mitchell Wortsman, and Ludwig Schmidt. 2023. OpenFlamingo: An Open-Source Framework for Training Large Autoregressive Vision-Language Models. arxiv: 2308.01390 [cs.CV]"},{"key":"e_1_3_2_1_4_1","volume-title":"Aaron Steven White, et al","author":"Barham Samuel","year":"2023","unstructured":"Samuel Barham, Orion Weller, Michelle Yuan, Kenton Murray, Mahsa Yarmohammadi, Zhengping Jiang, Siddharth Vashishtha, Alexander Martin, Anqi Liu, Aaron Steven White, et al. 2023. MegaWika: Millions of reports and their sources across 50 diverse languages. arXiv preprint arXiv:2307.07049 (2023)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2026"},{"key":"e_1_3_2_1_6_1","volume-title":"Jacob Eisenstein, Kuzman Ganchev, Jonathan Herzig, Kai Hui, Tom Kwiatkowski, Ji Ma, Jianmo Ni","author":"Bohnet Bernd","unstructured":"Bernd Bohnet, Vinh Q. Tran, Pat Verga, Roee Aharoni, Daniel Andor, Livio Baldini Soares, Jacob Eisenstein, Kuzman Ganchev, Jonathan Herzig, Kai Hui, Tom Kwiatkowski, Ji Ma, Jianmo Ni, Tal Schuster, William W. Cohen, Michael Collins, Dipanjan Das, Donald Metzler, Slav Petrov, and Kellie Webster. 2022. Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models. ArXiv , Vol. abs\/2212.08037 (2022). https:\/\/api.semanticscholar.org\/CorpusID:254685584"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1175"},{"key":"e_1_3_2_1_8_1","volume-title":"Neural machine translation with monolingual translation memory. arXiv preprint arXiv:2105.11269","author":"Cai Deng","year":"2021","unstructured":"Deng Cai, Yan Wang, Huayang Li, Wai Lam, and Lemao Liu. 2021. Neural machine translation with monolingual translation memory. arXiv preprint arXiv:2105.11269 (2021)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1015"},{"key":"e_1_3_2_1_10_1","volume-title":"BooookScore: A systematic exploration of book-length summarization in the era of LLMs. ArXiv","author":"Chang Yapei","year":"2023","unstructured":"Yapei Chang, Kyle Lo, Tanya Goyal, and Mohit Iyyer. 2023. BooookScore: A systematic exploration of book-length summarization in the era of LLMs. ArXiv , Vol. abs\/2310.00785 (2023). https:\/\/api.semanticscholar.org\/CorpusID:263605928"},{"key":"e_1_3_2_1_11_1","volume-title":"Benchmarking Large Language Models in Retrieval-Augmented Generation. ArXiv","author":"Chen Jiawei","year":"2023","unstructured":"Jiawei Chen, Hongyu Lin, Xianpei Han, and Le Sun. 2023. Benchmarking Large Language Models in Retrieval-Augmented Generation. ArXiv , Vol. abs\/2309.01431 (2023). https:\/\/api.semanticscholar.org\/CorpusID:261530434"},{"key":"e_1_3_2_1_12_1","volume-title":"MuRAG: Multimodal retrieval-augmented generator for open question answering over images and text. arXiv preprint arXiv:2210.02928","author":"Chen Wenhu","year":"2022","unstructured":"Wenhu Chen, Hexiang Hu, Xi Chen, Pat Verga, and William W Cohen. 2022. MuRAG: Multimodal retrieval-augmented generator for open question answering over images and text. arXiv preprint arXiv:2210.02928 (2022)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-1012"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331354"},{"key":"e_1_3_2_1_15_1","volume-title":"Overview of the TREC 2007 Question Answering Track.. In Trec","volume":"7","author":"Dang Hoa Trang","year":"2007","unstructured":"Hoa Trang Dang, Diane Kelly, Jimmy Lin, et al. 2007. Overview of the TREC 2007 Question Answering Track.. In Trec, Vol. 7. 63."},{"key":"e_1_3_2_1_16_1","volume-title":"Computational Processing of the Portuguese Language, Jorge Baptista, Nuno Mamede, Sara Candeias, Ivandr\u00e9 Paraboni, Thiago A. S. Pardo, and Maria das Gracc as Volpe Nunes (Eds.)","author":"Dias M\u00e1rcio","unstructured":"M\u00e1rcio de S. Dias, Val\u00e9ria D. Feltrim, and Thiago Alexandre Salgueiro Pardo. 2014. Using Rhetorical Structure Theory and Entity Grids to Automatically Evaluate Local Coherence in Texts. In Computational Processing of the Portuguese Language, Jorge Baptista, Nuno Mamede, Sara Candeias, Ivandr\u00e9 Paraboni, Thiago A. S. Pardo, and Maria das Gracc as Volpe Nunes (Eds.). Springer International Publishing, Cham, 232--243."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.81"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10489-022-03732--9"},{"key":"e_1_3_2_1_19_1","volume-title":"ELI5: Long Form Question Answering. arXiv preprint arXiv:1907.09190","author":"Fan Angela","year":"2019","unstructured":"Angela Fan, Yacine Jernite, Ethan Perez, David Grangier, Jason Weston, and Michael Auli. 2019. ELI5: Long Form Question Answering. arXiv preprint arXiv:1907.09190 (2019)."},{"key":"e_1_3_2_1_20_1","volume-title":"Neural machine translation with phrase-level universal visual representations. arXiv preprint arXiv:2203.10299","author":"Fang Qingkai","year":"2022","unstructured":"Qingkai Fang and Yang Feng. 2022. Neural machine translation with phrase-level universal visual representations. arXiv preprint arXiv:2203.10299 (2022)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3506860.3506977"},{"key":"e_1_3_2_1_22_1","article-title":"Academic Plagiarism Detection","volume":"52","author":"Folt\u00fdnek Tom\u00e1vs","year":"2019","unstructured":"Tom\u00e1vs Folt\u00fdnek, Norman Meuschke, and Bela Gipp. 2019. Academic Plagiarism Detection: A Systematic Literature Review. ACM Comput. Surv. , Vol. 52, 6, Article 112 (oct 2019), bibinfonumpages42 pages.","journal-title":"A Systematic Literature Review. ACM Comput. Surv."},{"key":"e_1_3_2_1_23_1","volume-title":"Enabling Large Language Models to Generate Text with Citations. ArXiv","author":"Gao Tianyu","year":"2023","unstructured":"Tianyu Gao, Ho-Ching Yen, Jiatong Yu, and Danqi Chen. 2023. Enabling Large Language Models to Generate Text with Citations. ArXiv , Vol. abs\/2305.14627 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258865710"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 23rd Conference on Computational Natural Language Learning (CoNLL), Mohit Bansal and Aline Villavicencio (Eds.). Association for Computational Linguistics","author":"Gao Yanjun","unstructured":"Yanjun Gao, Chen Sun, and Rebecca J. Passonneau. 2019. Automated Pyramid Summarization Evaluation. In Proceedings of the 23rd Conference on Computational Natural Language Learning (CoNLL), Mohit Bansal and Aline Villavicencio (Eds.). Association for Computational Linguistics, Hong Kong, China, 404--418. https:\/\/aclanthology.org\/K19--1038"},{"key":"e_1_3_2_1_25_1","volume-title":"Evaluating Generative Ad Hoc Information Retrieval. ArXiv","author":"Gienapp Lukas","year":"2023","unstructured":"Lukas Gienapp, Harrisen Scells, Niklas Deckers, Janek Bevendorff, Shuai Wang, Johannes Kiesel, Shahbaz Syed, Maik Frobe, Guide Zucoon, Benno Stein, Matthias Hagen, and Martin Potthast. 2023. Evaluating Generative Ad Hoc Information Retrieval. ArXiv , Vol. abs\/2311.04694 (2023). https:\/\/api.semanticscholar.org\/CorpusID:265050661"},{"key":"e_1_3_2_1_26_1","volume-title":"Lucy Lu Wang, and Arman Cohan","author":"Giorgi John","year":"2023","unstructured":"John Giorgi, Luca Soldaini, Bo Wang, Gary Bader, Kyle Lo, Lucy Lu Wang, and Arman Cohan. 2023. Open Domain Multi-document Summarization: A Comprehensive Study of Model Brittleness under Retrieval. In Findings of the Association for Computational Linguistics: EMNLP 2023. 8177--8199."},{"key":"e_1_3_2_1_27_1","volume-title":"Junyi Jessy Li, and Greg Durrett","author":"Goyal Tanya","year":"2022","unstructured":"Tanya Goyal, Junyi Jessy Li, and Greg Durrett. 2022. News Summarization and Evaluation in the Era of GPT-3. ArXiv , Vol. abs\/2209.12356 (2022). https:\/\/api.semanticscholar.org\/CorpusID:252532176"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1013"},{"key":"e_1_3_2_1_29_1","volume-title":"TREC 2016 Total Recall Track Overview.. In TREC.","author":"Grossman Maura R","year":"2016","unstructured":"Maura R Grossman, Gordon V Cormack, and Adam Roegiest. 2016. TREC 2016 Total Recall Track Overview.. In TREC."},{"key":"e_1_3_2_1_30_1","volume-title":"REALM: Retrieval-Augmented Language Model Pre-Training. ArXiv","author":"Guu Kelvin","year":"2020","unstructured":"Kelvin Guu, Kenton Lee, Zora Tung, Panupong Pasupat, and Ming-Wei Chang. 2020. REALM: Retrieval-Augmented Language Model Pre-Training. ArXiv , Vol. abs\/2002.08909 (2020). https:\/\/api.semanticscholar.org\/CorpusID:211204736"},{"key":"e_1_3_2_1_31_1","volume-title":"Text Summarization Branches Out","author":"Harman Donna","unstructured":"Donna Harman and Paul Over. 2004. The Effects of Human Variation in DUC Summarization Evaluation. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 10--17. https:\/\/aclanthology.org\/W04--1003"},{"key":"e_1_3_2_1_32_1","volume-title":"TRUE: Re-evaluating Factual Consistency Evaluation. In Proceedings of the 2022 Conference of the North American","author":"Honovich Or","year":"2022","unstructured":"Or Honovich, Roee Aharoni, Jonathan Herzig, Hagai Taitelbaum, Doron Kukliansy, Vered Cohen, Thomas Scialom, Idan Szpektor, Avinatan Hassidim, and Yossi Matias. 2022. TRUE: Re-evaluating Factual Consistency Evaluation. In Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, , Marine Carpuat, Marie-Catherine de Marneffe, and Ivan Vladimir Meza Ruiz (Eds.). Association for Computational Linguistics, Seattle, United States, 3905--3920. https:\/\/aclanthology.org\/2022.naacl-main.287"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02238"},{"key":"e_1_3_2_1_34_1","volume-title":"Imtiaz Hussain Khan, and Muazzam Ahmed Siddiqui","author":"Jambi Kamal Mansour","year":"2022","unstructured":"Kamal Mansour Jambi, Imtiaz Hussain Khan, and Muazzam Ahmed Siddiqui. 2022. Evaluation of Different Plagiarism Detection Methods: A Fuzzy MCDM Perspective. Applied Sciences, Vol. 12, 9 (2022). https:\/\/www.mdpi.com\/2076--3417\/12\/9\/4580"},{"key":"e_1_3_2_1_35_1","volume-title":"ACM Trans. Inf. Syst.","volume":"20","author":"Jaana Kalervo","year":"2002","unstructured":"Kalervo J\"arvelin and Jaana Kek\"al\"ainen. 2002. Cumulated gain-based evaluation of IR techniques. ACM Trans. Inf. Syst. , Vol. 20, 4 (oct 2002), 422--446."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.1096.abs"},{"key":"e_1_3_2_1_37_1","unstructured":"Judith L Klavans Min-yen Kan and Kathleen McKeown. 2001. Domain-specific informative and indicative summarization for information retrieval. In SIGIR Worshop on Text Summarization."},{"key":"e_1_3_2_1_38_1","volume-title":"Internet-augmented dialogue generation. arXiv preprint arXiv:2107.07566","author":"Komeili Mojtaba","year":"2021","unstructured":"Mojtaba Komeili, Kurt Shuster, and Jason Weston. 2021. Internet-augmented dialogue generation. arXiv preprint arXiv:2107.07566 (2021)."},{"key":"e_1_3_2_1_39_1","volume-title":"Hurdles to Progress in Long-form Question Answering. arXiv preprint arXiv:2103.06332","author":"Krishna Kalpesh","year":"2021","unstructured":"Kalpesh Krishna, Aurko Roy, and Mohit Iyyer. 2021. Hurdles to Progress in Long-form Question Answering. arXiv preprint arXiv:2103.06332 (2021)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.5555\/1642293.1642467"},{"key":"e_1_3_2_1_41_1","first-page":"9459","article-title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen tau Yih, Tim Rockt\u00e4schel, Sebastian Riedel, and Douwe Kiela. 2020. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks. Advances in Neural Information Processing Systems , Vol. 33 (2020), 9459--9474.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3133015"},{"key":"e_1_3_2_1_43_1","volume-title":"A survey on retrieval-augmented text generation. arXiv preprint arXiv:2202.01110","author":"Li Huayang","year":"2022","unstructured":"Huayang Li, Yixuan Su, Deng Cai, Yan Wang, and Lemao Liu. 2022. A survey on retrieval-augmented text generation. arXiv preprint arXiv:2202.01110 (2022)."},{"key":"e_1_3_2_1_44_1","volume-title":"retrieve, generate: a simple approach to sentiment and style transfer. arXiv preprint arXiv:1804.06437","author":"Li Juncen","year":"2018","unstructured":"Juncen Li, Robin Jia, He He, and Percy Liang. 2018. Delete, retrieve, generate: a simple approach to sentiment and style transfer. arXiv preprint arXiv:1804.06437 (2018)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.551"},{"key":"e_1_3_2_1_46_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74--81. https:\/\/aclanthology.org\/W04--1013"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073160"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073445.1073465"},{"key":"e_1_3_2_1_49_1","volume-title":"Retrieval augmented visual question answering with outside knowledge. arXiv preprint arXiv:2210.03809","author":"Lin Weizhe","year":"2022","unstructured":"Weizhe Lin and Bill Byrne. 2022. Retrieval augmented visual question answering with outside knowledge. arXiv preprint arXiv:2210.03809 (2022)."},{"key":"e_1_3_2_1_50_1","volume-title":"Hwee Tou Ng, and Min-Yen Kan","author":"Lin Ziheng","year":"2011","unstructured":"Ziheng Lin, Hwee Tou Ng, and Min-Yen Kan. 2011. Automatically Evaluating Text Coherence Using Discourse Relations. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies, Dekang Lin, Yuji Matsumoto, and Rada Mihalcea (Eds.). Association for Computational Linguistics, Portland, Oregon, USA, 997--1006. https:\/\/aclanthology.org\/P11--1100"},{"key":"e_1_3_2_1_51_1","unstructured":"Haotian Liu Chunyuan Li Qingyang Wu and Yong Jae Lee. 2023 a. Visual Instruction Tuning. arxiv: 2304.08485 [cs.CV]"},{"key":"e_1_3_2_1_52_1","unstructured":"Jerry Liu. 2022. LlamaIndex. https:\/\/github.com\/jerryjliu\/llama_index"},{"key":"e_1_3_2_1_53_1","volume-title":"2023 b. Evaluating Verifiability in Generative Search Engines. ArXiv","author":"Liu Nelson F.","year":"2023","unstructured":"Nelson F. Liu, Tianyi Zhang, and Percy Liang. 2023 b. Evaluating Verifiability in Generative Search Engines. ArXiv , Vol. abs\/2304.09848 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258212854"},{"key":"e_1_3_2_1_54_1","volume-title":"Annual Meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/CorpusID:254877603","author":"Mallen Alex","year":"2022","unstructured":"Alex Mallen, Akari Asai, Victor Zhong, Rajarshi Das, Hannaneh Hajishirzi, and Daniel Khashabi. 2022. When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories. In Annual Meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/CorpusID:254877603"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1075\/nlp.3"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661951"},{"key":"e_1_3_2_1_57_1","volume-title":"2023 a. SILO Language Models: Isolating Legal Risk In a Nonparametric Datastore. ArXiv","author":"Min Sewon","year":"2023","unstructured":"Sewon Min, Suchin Gururangan, Eric Wallace, Hannaneh Hajishirzi, Noah A. Smith, and Luke Zettlemoyer. 2023 a. SILO Language Models: Isolating Legal Risk In a Nonparametric Datastore. ArXiv , Vol. abs\/2308.04430 (2023). https:\/\/api.semanticscholar.org\/CorpusID:260704206"},{"key":"e_1_3_2_1_58_1","volume-title":"Pang Wei Koh, Mohit Iyyer, Luke Zettlemoyer, and Hannaneh Hajishirzi. 2023 b. FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation. ArXiv","author":"Min Sewon","year":"2023","unstructured":"Sewon Min, Kalpesh Krishna, Xinxi Lyu, Mike Lewis, Wen tau Yih, Pang Wei Koh, Mohit Iyyer, Luke Zettlemoyer, and Hannaneh Hajishirzi. 2023 b. FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation. ArXiv , Vol. abs\/2305.14251 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258841470"},{"key":"e_1_3_2_1_59_1","volume-title":"Overview of TAC KBP 2016 Event Nugget Track.. In TAC.","author":"Mitamura Teruko","year":"2016","unstructured":"Teruko Mitamura, Zhengzhong Liu, and Eduard H Hovy. 2016. Overview of TAC KBP 2016 Event Nugget Track.. In TAC."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2023.03.057"},{"key":"e_1_3_2_1_61_1","volume-title":"Proceedings of the Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics: HLT-NAACL","author":"Nenkova Ani","year":"2004","unstructured":"Ani Nenkova and Rebecca Passonneau. 2004. Evaluating Content Selection in Summarization: The Pyramid Method. In Proceedings of the Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics: HLT-NAACL 2004. Association for Computational Linguistics, Boston, Massachusetts, USA, 145--152. https:\/\/aclanthology.org\/N04--1019"},{"key":"e_1_3_2_1_62_1","unstructured":"Richard Yuanzhe Pang Alicia Parrish Nitish Joshi Nikita Nangia Jason Phang Angelica Chen Vishakh Padmakumar Johnny Ma Jana Thompson He He et al. 2021. QuALITY: Question Answering with Long Input Texts Yes! arXiv preprint arXiv:2112.08608 (2021)."},{"key":"e_1_3_2_1_63_1","volume-title":"Continual BERT: Continual learning for adaptive extractive summarization of COVID-19 literature. arXiv preprint arXiv:2007.03405","author":"Park Jong Won","year":"2020","unstructured":"Jong Won Park. 2020. Continual BERT: Continual learning for adaptive extractive summarization of COVID-19 literature. arXiv preprint arXiv:2007.03405 (2020)."},{"key":"e_1_3_2_1_64_1","volume-title":"Text generation with exemplar-based adaptive decoding. arXiv preprint arXiv:1904.04428","author":"Peng Hao","year":"2019","unstructured":"Hao Peng, Ankur P Parikh, Manaal Faruqui, Bhuwan Dhingra, and Dipanjan Das. 2019. Text generation with exemplar-based adaptive decoding. arXiv preprint arXiv:1904.04428 (2019)."},{"key":"e_1_3_2_1_65_1","volume-title":"MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers. In NeurIPS.","author":"Pillutla Krishna","year":"2021","unstructured":"Krishna Pillutla, Swabha Swayamdipta, Rowan Zellers, John Thickstun, Sean Welleck, Yejin Choi, and Zaid Harchaoui. 2021. MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers. In NeurIPS."},{"key":"e_1_3_2_1_66_1","volume-title":"Alberto Barr\u00f3n-Cede no, and Paolo Rosso","author":"Potthast Martin","year":"2010","unstructured":"Martin Potthast, Benno Stein, Alberto Barr\u00f3n-Cede no, and Paolo Rosso. 2010. An Evaluation Framework for Plagiarism Detection. In Coling 2010: Posters, , Chu-Ren Huang and Dan Jurafsky (Eds.). Coling 2010 Organizing Committee, Beijing, China, 997--1005. https:\/\/aclanthology.org\/C10--2115"},{"key":"e_1_3_2_1_67_1","volume-title":"Recognizing textual entailment: A review of resources, approaches, applications, and challenges","author":"Putra I Made Suwija","year":"2023","unstructured":"I Made Suwija Putra, Daniel Siahaan, and Ahmad Saikhu. 2023. Recognizing textual entailment: A review of resources, approaches, applications, and challenges. ICT Express (2023). https:\/\/www.sciencedirect.com\/science\/article\/pii\/S2405959523001145"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409837"},{"key":"e_1_3_2_1_69_1","volume-title":"100,000 Questions for Machine Comprehension of Text. arXiv preprint arXiv:1606.05250","author":"Rajpurkar Pranav","year":"2016","unstructured":"Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, and Percy Liang. 2016. SQuAD: 100,000 Questions for Machine Comprehension of Text. arXiv preprint arXiv:1606.05250 (2016)."},{"key":"e_1_3_2_1_70_1","volume-title":"Iulia Turc, and David Reitter.","author":"Rashkin Hannah","year":"2023","unstructured":"Hannah Rashkin, Vitaly Nikolaev, Matthew Lamm, Lora Aroyo, Michael Collins, Dipanjan Das, Slav Petrov, Gaurav Singh Tomar, Iulia Turc, and David Reitter. 2023. Measuring attribution in natural language generation models. Computational Linguistics (2023), 1--64."},{"key":"e_1_3_2_1_71_1","volume-title":"Sentence-BERT: Sentence embeddings using siamese BERT-networks. arXiv preprint arXiv:1908.10084","author":"Reimers Nils","year":"2019","unstructured":"Nils Reimers and Iryna Gurevych. 2019. Sentence-BERT: Sentence embeddings using siamese BERT-networks. arXiv preprint arXiv:1908.10084 (2019)."},{"key":"e_1_3_2_1_72_1","volume-title":"TREC 2015 Total Recall Track Overview.. In TREC.","author":"Roegiest Adam","year":"2015","unstructured":"Adam Roegiest, Gordon V Cormack, Charles LA Clarke, and Maura R Grossman. 2015. TREC 2015 Total Recall Track Overview.. In TREC."},{"key":"e_1_3_2_1_73_1","volume-title":"ARES: An automated evaluation framework for retrieval-augmented generation systems. arXiv preprint arXiv:2311.09476","author":"Saad-Falcon Jon","year":"2023","unstructured":"Jon Saad-Falcon, Omar Khattab, Christopher Potts, and Matei Zaharia. 2023. ARES: An automated evaluation framework for retrieval-augmented generation systems. arXiv preprint arXiv:2311.09476 (2023)."},{"key":"e_1_3_2_1_74_1","volume-title":"Biennial Conference on Design of Experimental Search & Information Retrieval Systems. https:\/\/api.semanticscholar.org\/CorpusID:238207962","author":"David","unstructured":"David P. Sander and Laura Dietz. 2021. EXAM: How to Evaluate Retrieve-and-Generate Systems for Users Who Do Not (Yet) Know What They Want. In Biennial Conference on Design of Experimental Search & Information Retrieval Systems. https:\/\/api.semanticscholar.org\/CorpusID:238207962"},{"key":"e_1_3_2_1_75_1","volume-title":"BLEURT: Learning robust metrics for text generation. arXiv preprint arXiv:2004.04696","author":"Sellam Thibault","year":"2020","unstructured":"Thibault Sellam, Dipanjan Das, and Ankur P Parikh. 2020. BLEURT: Learning robust metrics for text generation. arXiv preprint arXiv:2004.04696 (2020)."},{"key":"e_1_3_2_1_76_1","volume-title":"Luis Espinosa Anke, and Steven Schockaert","author":"Shahul ES","year":"2023","unstructured":"ES Shahul, Jithin James, Luis Espinosa Anke, and Steven Schockaert. 2023. RAGAS: Automated Evaluation of Retrieval Augmented Generation. ArXiv , Vol. abs\/2309.15217 (2023). https:\/\/api.semanticscholar.org\/CorpusID:263152733"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"crossref","unstructured":"Kurt Shuster Spencer Poff Moya Chen Douwe Kiela and Jason Weston. 2021. Retrieval Augmentation Reduces Hallucination in Conversation. arxiv: 2104.07567 [cs.CL]","DOI":"10.18653\/v1\/2021.findings-emnlp.320"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_79_1","volume-title":"Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC","author":"Song Zhiyi","year":"2018","unstructured":"Zhiyi Song, Ann Bies, Justin Mott, Xuansong Li, Stephanie Strassel, and Christopher Caruso. 2018. Cross-document, cross-language event coreference annotation using event hoppers. In Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018)."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073445.1073475"},{"key":"e_1_3_2_1_81_1","volume-title":"Report on the Need for and Provision of an `ideal' Information Retrieval Test Collection","author":"Jones Karen Sparck","unstructured":"Karen Sparck Jones and C.J. Van Rijsbergen. 1975. Report on the Need for and Provision of an `ideal' Information Retrieval Test Collection. University Computer Laboratory."},{"key":"e_1_3_2_1_82_1","volume-title":"Odunayo Ogundepo, Ehsan Kamalloo, David Alfonso-Hermelo, Xiaoguang Li, Qun Liu, Boxing Chen, Mehdi Rezagholizadeh, and Jimmy J. Lin.","author":"Thakur Nandan","year":"2023","unstructured":"Nandan Thakur, Luiz Bonifacio, Xinyu Crystina Zhang, Odunayo Ogundepo, Ehsan Kamalloo, David Alfonso-Hermelo, Xiaoguang Li, Qun Liu, Boxing Chen, Mehdi Rezagholizadeh, and Jimmy J. Lin. 2023. NoMIRACL: Knowing When You Don't Know for Robust Multilingual Retrieval-Augmented Generation. ArXiv , Vol. abs\/2312.11361 (2023). https:\/\/api.semanticscholar.org\/CorpusID:266359301"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2007.01.023"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1017\/S1351324901002789"},{"key":"e_1_3_2_1_85_1","volume-title":"The Text REtrieval Conferences (TRECs). In TIPSTER TEXT PROGRAM PHASE III: Proceedings of a Workshop held at Baltimore, Maryland, October 13--15","author":"Ellen","year":"1999","unstructured":"Ellen M. Voorhees and Donna Harman. 1998. The Text REtrieval Conferences (TRECs). In TIPSTER TEXT PROGRAM PHASE III: Proceedings of a Workshop held at Baltimore, Maryland, October 13--15, 1999. Association for Computational Linguistics, Baltimore, Maryland, USA, 241--273. https:\/\/aclanthology.org\/X98--1031"},{"key":"e_1_3_2_1_86_1","volume-title":"Bowman","author":"Wang Alex","year":"2019","unstructured":"Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R. Bowman. 2019a. SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems. arXiv preprint 1905.00537 (2019)."},{"key":"e_1_3_2_1_87_1","volume-title":"Bowman","author":"Wang Alex","year":"2019","unstructured":"Alex Wang, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R. Bowman. 2019b. GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding. In the Proceedings of ICLR."},{"key":"e_1_3_2_1_88_1","volume-title":"A Critical Evaluation of Evaluations for Long-form Question Answering. arXiv preprint arXiv:2305.18201","author":"Xu Fangyuan","year":"2023","unstructured":"Fangyuan Xu, Yixiao Song, Mohit Iyyer, and Eunsol Choi. 2023. A Critical Evaluation of Evaluations for Long-form Question Answering. arXiv preprint arXiv:2305.18201 (2023)."},{"key":"e_1_3_2_1_89_1","volume-title":"Retrieval-augmented multimodal language modeling. arXiv preprint arXiv:2211.12561","author":"Yasunaga Michihiro","year":"2022","unstructured":"Michihiro Yasunaga, Armen Aghajanyan, Weijia Shi, Rich James, Jure Leskovec, Percy Liang, Mike Lewis, Luke Zettlemoyer, and Wen-tau Yih. 2022. Retrieval-augmented multimodal language modeling. arXiv preprint arXiv:2211.12561 (2022)."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.307"},{"key":"e_1_3_2_1_91_1","volume-title":"BERTScore: Evaluating text generation with BERT. arXiv preprint arXiv:1904.09675","author":"Zhang Tianyi","year":"2019","unstructured":"Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q Weinberger, and Yoav Artzi. 2019. BERTScore: Evaluating text generation with BERT. arXiv preprint arXiv:1904.09675 (2019)."},{"key":"e_1_3_2_1_92_1","volume-title":"ODSum: New Benchmarks for Open Domain Multi-Document Summarization. arXiv preprint arXiv:2309.08960","author":"Zhou Yijie","year":"2023","unstructured":"Yijie Zhou, Kejian Shi, Wencai Zhang, Yixin Liu, Yilun Zhao, and Arman Cohan. 2023. ODSum: New Benchmarks for Open Domain Multi-Document Summarization. arXiv preprint arXiv:2309.08960 (2023)."},{"key":"e_1_3_2_1_93_1","volume-title":"Miguel Eckstein, and William Yang Wang.","author":"Zhu Wanrong","year":"2022","unstructured":"Wanrong Zhu, An Yan, Yujie Lu, Wenda Xu, Xin Eric Wang, Miguel Eckstein, and William Yang Wang. 2022. Visualize Before You Write: Imagination-Guided Open-Ended Text Generation. arXiv preprint arXiv:2210.03765 (2022)."},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657846","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657846","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:43:40Z","timestamp":1755841420000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657846"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":94,"alternative-id":["10.1145\/3626772.3657846","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657846","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}