{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T02:35:22Z","timestamp":1780626922092,"version":"3.54.1"},"reference-count":73,"publisher":"MIT Press","license":[{"start":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T00:00:00Z","timestamp":1706832000000},"content-version":"vor","delay-in-days":32,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,31]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>Large language models (LLMs) have shown promise for automatic summarization but the reasons behind their successes are poorly understood. By conducting a human evaluation on ten LLMs across different pretraining methods, prompts, and model scales, we make two important observations. First, we find instruction tuning, not model size, is the key to the LLM\u2019s zero-shot summarization capability. Second, existing studies have been limited by low-quality references, leading to underestimates of human performance and lower few-shot and finetuning performance. To better evaluate LLMs, we perform human evaluation over high-quality summaries we collect from freelance writers. Despite major stylistic differences such as the amount of paraphrasing, we find that LLM summaries are judged to be on par with human written summaries.<\/jats:p>","DOI":"10.1162\/tacl_a_00632","type":"journal-article","created":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T23:51:21Z","timestamp":1706917881000},"page":"39-57","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":301,"title":["Benchmarking Large Language Models for News Summarization"],"prefix":"10.1162","volume":"12","author":[{"given":"Tianyi","family":"Zhang","sequence":"first","affiliation":[{"name":"Stanford University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Faisal","family":"Ladhak","sequence":"additional","affiliation":[{"name":"Columbia University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Esin","family":"Durmus","sequence":"additional","affiliation":[{"name":"Stanford University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Percy","family":"Liang","sequence":"additional","affiliation":[{"name":"Stanford University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kathleen","family":"McKeown","sequence":"additional","affiliation":[{"name":"Columbia University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tatsunori B.","family":"Hashimoto","sequence":"additional","affiliation":[{"name":"Stanford University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"281","published-online":{"date-parts":[[2024,1,31]]},"reference":[{"key":"2024020223510482400_bib1","article-title":"Neural machine translation by jointly learning to align and translate","volume-title":"3rd International Conference on Learning Representations, ICLR 2015","author":"Bahdanau","year":"2015"},{"key":"2024020223510482400_bib2","article-title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","author":"Bai","year":"2022","journal-title":"arXiv"},{"key":"2024020223510482400_bib3","article-title":"Meteor: An automatic metric for mt evaluation with improved correlation with human judgments","volume-title":"IEEvaluation@ACL","author":"Banerjee","year":"2005"},{"key":"2024020223510482400_bib4","article-title":"Using lexical chains for text summarization","volume-title":"Proceedings of ISTS, ACL 1997","author":"Barzilay","year":"1997"},{"issue":"3","key":"2024020223510482400_bib5","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1162\/089120105774321091","article-title":"Sentence fusion for multidocument news summarization","volume":"31","author":"Barzilay","year":"2005","journal-title":"Computational Linguistics"},{"key":"2024020223510482400_bib6","doi-asserted-by":"publisher","first-page":"550","DOI":"10.3115\/1034678.1034760","article-title":"Information fusion in the context of multi-document summarization","volume-title":"Proceedings of the 37th Annual Meeting of the Association for Computational Linguistics","author":"Barzilay","year":"1999"},{"key":"2024020223510482400_bib7","article-title":"On the opportunities and risks of foundation models","author":"Bommasani","year":"2021","journal-title":"arXiv preprint arXiv:2108.07258"},{"key":"2024020223510482400_bib8","article-title":"Language models are few-shot learners","volume-title":"NeurIPS","author":"Brown","year":"2020"},{"key":"2024020223510482400_bib9","doi-asserted-by":"publisher","first-page":"675","DOI":"10.18653\/v1\/P18-1063","article-title":"Fast abstractive summarization with reinforce-selected sentence rewriting","volume-title":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Chen","year":"2018"},{"key":"2024020223510482400_bib10","article-title":"Palm: Scaling language modeling with pathways","author":"Chowdhery","year":"2022","journal-title":"arXiv preprint arXiv:2204.02311"},{"key":"2024020223510482400_bib11","article-title":"Introduction to large language models","author":"Cohere","year":"2022"},{"key":"2024020223510482400_bib12","doi-asserted-by":"publisher","first-page":"137","DOI":"10.3115\/1599081.1599099","article-title":"Sentence compression beyond word deletion","volume-title":"Proceedings of the 22nd International Conference on Computational Linguistics (Coling 2008)","author":"Cohn","year":"2008"},{"key":"2024020223510482400_bib13","doi-asserted-by":"publisher","DOI":"10.12968\/sece.2006.11.755s","article-title":"Back to basics: Classy 2006","volume-title":"Proceedings of the Document Understanding Conference","author":"Conroy","year":"2006"},{"key":"2024020223510482400_bib14","doi-asserted-by":"publisher","first-page":"300","DOI":"10.18653\/v1\/2021.conll-1.24","article-title":"Understanding the extent to which content quality metrics measure the information quality of summaries","volume-title":"Proceedings of the 25th Conference on Computational Natural Language Learning","author":"Deutsch","year":"2021"},{"key":"2024020223510482400_bib15","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1905.03197","article-title":"Unified language model pre-training for natural language understanding and generation","author":"Li","year":"2019"},{"key":"2024020223510482400_bib16","article-title":"Glm: General language model pretraining with autoregressive blank infilling","volume-title":"ACL","author":"Zhengxiao","year":"2021"},{"key":"2024020223510482400_bib17","doi-asserted-by":"publisher","first-page":"5055","DOI":"10.18653\/v1\/2020.acl-main.454","article-title":"FEQA: A question answering evaluation framework for faithfulness assessment in abstractive summarization","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Durmus","year":"2020"},{"key":"2024020223510482400_bib18","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.102","article-title":"Spurious correlations in reference-free evaluation of text generation","volume-title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Durmus","year":"2022"},{"key":"2024020223510482400_bib19","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1523","article-title":"Lexrank: Graph-based centrality as salience in text summarization","author":"Erkan","year":"2004","journal-title":"Journal of Artificial Intelligence Research"},{"key":"2024020223510482400_bib20","doi-asserted-by":"publisher","first-page":"2587","DOI":"10.18653\/v1\/2022.naacl-main.187","article-title":"QAFactEval: Improved QA-based factual consistency evaluation for summarization","volume-title":"Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Fabbri","year":"2022"},{"key":"2024020223510482400_bib21","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00373","article-title":"Summeval: Re-evaluating summarization evaluation","author":"Fabbri","year":"2020","journal-title":"arXiv preprint arXiv:2007.12626"},{"key":"2024020223510482400_bib22","doi-asserted-by":"publisher","first-page":"177","DOI":"10.3115\/1613715.1613741","article-title":"Sentence fusion via dependency graph compression","volume-title":"Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing","author":"Filippova","year":"2008"},{"key":"2024020223510482400_bib23","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.5","article-title":"BLEU might be guilty but references are not innocent","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Freitag","year":"2020"},{"key":"2024020223510482400_bib24","doi-asserted-by":"publisher","first-page":"3592","DOI":"10.18653\/v1\/2020.findings-emnlp.322","article-title":"Evaluating factuality in generation with dependency-level entailment","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Goyal","year":"2020"},{"key":"2024020223510482400_bib25","article-title":"News summarization and evaluation in the era of gpt-3","author":"Goyal","year":"2022","journal-title":"ArXiv"},{"key":"2024020223510482400_bib26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1065","article-title":"Newsroom: A dataset of 1.3 million summaries with diverse extractive strategies","volume-title":"North American Chapter of the Association for Computational Linguistics","author":"Grusky","year":"2018"},{"key":"2024020223510482400_bib27","article-title":"Teaching machines to read and comprehend","volume-title":"NeurIPS","author":"Hermann","year":"2015"},{"key":"2024020223510482400_bib28","first-page":"82","article-title":"Automated text summarization in summarist","volume-title":"Advances in Automatic Text Summarization","author":"Hovy","year":"1999"},{"key":"2024020223510482400_bib29","doi-asserted-by":"publisher","DOI":"10.3115\/974147.974190","article-title":"Sentence reduction for automatic text summarization","volume-title":"Applied Natural Language Processing Conference","author":"Jing","year":"2000"},{"key":"2024020223510482400_bib30","article-title":"Cut and paste based text summarization","volume-title":"Applied Natural Language Processing Conference","author":"Jing","year":"2000"},{"key":"2024020223510482400_bib31","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1145\/312624.312666","article-title":"The decomposition of human-written summary sentences","volume-title":"Proceedings of the 22nd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Jing","year":"1999"},{"key":"2024020223510482400_bib32","doi-asserted-by":"publisher","first-page":"718","DOI":"10.18653\/v1\/2020.acl-main.66","article-title":"Improved natural language generation via loss truncation","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Kang","year":"2020"},{"issue":"1","key":"2024020223510482400_bib33","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1016\/S0004-3702(02)00222-9","article-title":"Summarization beyond sentence extraction: A probabilistic approach to sentence compression","volume":"139","author":"Knight","year":"2002","journal-title":"Artificial Intelligence"},{"key":"2024020223510482400_bib34","doi-asserted-by":"publisher","first-page":"193","DOI":"10.3115\/1557690.1557745","article-title":"Query-based sentence fusion is better defined and leads to more preferred results than generic sentence fusion","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","author":"Krahmer","year":"2008"},{"key":"2024020223510482400_bib35","doi-asserted-by":"publisher","first-page":"163","DOI":"10.1162\/tacl_a_00453","article-title":"Summac: Re-visiting nli-based models for inconsistency detection in summarization","volume":"10","author":"Laban","year":"2021","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024020223510482400_bib36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703","article-title":"Bart: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","volume-title":"Annual Meeting of the Association for Computational Linguistics","author":"Lewis","year":"2019"},{"key":"2024020223510482400_bib37","article-title":"Holistic evaluation of language models","author":"Liang","year":"2022","journal-title":"arXiv preprint arXiv:2211.09110"},{"key":"2024020223510482400_bib38","doi-asserted-by":"publisher","first-page":"457","DOI":"10.3115\/1073083.1073160","article-title":"From single to multi-document summarization: A prototype system and its evaluation","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","author":"Lin","year":"2002"},{"key":"2024020223510482400_bib39","article-title":"Rouge: A package for automatic evaluation of summaries","volume-title":"Annual Meeting of the Association for Computational Linguistics","author":"Lin","year":"2004"},{"key":"2024020223510482400_bib40","doi-asserted-by":"publisher","first-page":"3730","DOI":"10.18653\/v1\/D19-1387","article-title":"Text summarization with pretrained encoders","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Liu","year":"2019"},{"key":"2024020223510482400_bib41","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.228","article-title":"Revisiting the gold standard: Grounding summarization evaluation with robust human evaluation","author":"Liu","year":"2022","journal-title":"ArXiv"},{"key":"2024020223510482400_bib42","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.207","article-title":"Brio: Bringing order to abstractive summarization","volume-title":"Annual Meeting of the Association for Computational Linguistics","author":"Liu","year":"2022"},{"issue":"1\u20132","key":"2024020223510482400_bib43","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1023\/A:1009930203452","article-title":"Summarizing similarities and differences among related documents","volume":"1","author":"Mani","year":"1999","journal-title":"Information Retrieval"},{"key":"2024020223510482400_bib44","article-title":"From discourse structures to text summaries","volume-title":"Intelligent Scalable Text Summarization","author":"Marcu","year":"1997"},{"key":"2024020223510482400_bib45","first-page":"109","article-title":"Explorations in sentence fusion","volume-title":"Proceedings of the European Workshop on Natural Language Generation 2005","author":"Marsi","year":"2005"},{"key":"2024020223510482400_bib46","doi-asserted-by":"publisher","first-page":"1906","DOI":"10.18653\/v1\/2020.acl-main.173","article-title":"On faithfulness and factuality in abstractive summarization","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Maynez","year":"2020"},{"key":"2024020223510482400_bib47","first-page":"297","article-title":"Discriminative sentence compression with soft syntactic evidence","volume-title":"11th Conference of the European Chapter of the Association for Computational Linguistics","author":"McDonald","year":"2006"},{"key":"2024020223510482400_bib48","article-title":"Multi-document summarization with iterative graph-based algorithms","volume-title":"Proceedings of the First International Conference on Intelligent Analysis Methods and Tools (IA 2005)","author":"Mihalcea","year":"2005"},{"key":"2024020223510482400_bib49","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1206","article-title":"Don\u2019t give me the details, just the summary! Topic-aware convolutional neural networks for extreme summarization","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Narayan","year":"2018"},{"issue":"2\u20133","key":"2024020223510482400_bib50","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1561\/1500000015","article-title":"Automatic summarization","volume":"52","author":"Nenkova","year":"2011","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"2024020223510482400_bib51","doi-asserted-by":"publisher","first-page":"573","DOI":"10.1145\/1148170.1148269","article-title":"A compositional context sensitive multi-document summarizer: Exploring the factors that influence summarization","volume-title":"Proceedings of the Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Nenkova","year":"2006"},{"key":"2024020223510482400_bib52","article-title":"Training language models to follow instructions with human feedback","author":"Ouyang","year":"2022","journal-title":"arXiv preprint arXiv:2203.02155"},{"key":"2024020223510482400_bib53","doi-asserted-by":"publisher","first-page":"399","DOI":"10.1162\/089120102762671927","article-title":"Introduction to the special issue on summarization","volume":"28","author":"Radev","year":"2002","journal-title":"Computational Linguistics"},{"key":"2024020223510482400_bib54","article-title":"Centroid-based summarization of multiple documents: Sentence extraction, utility-based evaluation, and user studies","volume-title":"NAACL-ANLP 2000 Workshop: Automatic Summarization","author":"Radev","year":"2000"},{"key":"2024020223510482400_bib55","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1044","article-title":"A neural attention model for abstractive sentence summarization","author":"Rush","year":"2015","journal-title":"Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing"},{"issue":"2","key":"2024020223510482400_bib56","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1016\/S0306-4573(96)00062-3","article-title":"Automatic text structuring and summarization","volume":"33","author":"Salton","year":"1997","journal-title":"Information Processing & Management"},{"key":"2024020223510482400_bib57","article-title":"Multitask prompted training enables zero-shot task generalization","author":"Sanh","year":"2021","journal-title":"arXiv preprint arXiv:2110.08207"},{"key":"2024020223510482400_bib58","doi-asserted-by":"publisher","first-page":"1073","DOI":"10.18653\/v1\/P17-1099","article-title":"Get to the point: Summarization with pointer-generator networks","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"See","year":"2017"},{"key":"2024020223510482400_bib59","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.704","article-title":"Bleurt: Learning robust metrics for text generation","volume-title":"Annual Meeting of the Association for Computational Linguistics","author":"Sellam","year":"2020"},{"issue":"4","key":"2024020223510482400_bib60","doi-asserted-by":"publisher","first-page":"487","DOI":"10.1162\/089120102762671954","article-title":"Efficiently computed lexical chains as an intermediate representation for automatic text summarization","volume":"28","author":"Grogory Silber","year":"2002","journal-title":"Computational Linguistics"},{"issue":"6","key":"2024020223510482400_bib61","doi-asserted-by":"publisher","first-page":"1663","DOI":"10.1016\/j.ipm.2007.01.010","article-title":"Two uses of anaphora resolution in summarization","volume":"43","author":"Steinberger","year":"2007","journal-title":"Information Processing and Management"},{"key":"2024020223510482400_bib62","article-title":"Learning to summarize from human feedback","author":"Stiennon","year":"2020","journal-title":"arXiv preprint arXiv:2009.01325"},{"key":"2024020223510482400_bib63","article-title":"Sequence to sequence learning with neural networks","volume":"27","author":"Sutskever","year":"2014","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2024020223510482400_bib64","article-title":"Supervised sentence fusion with single-stage inference","volume-title":"Proceedings of IJCNLP","author":"Thadani","year":"2013"},{"key":"2024020223510482400_bib65","doi-asserted-by":"publisher","first-page":"11","DOI":"10.18653\/v1\/2020.eval4nlp-1.2","article-title":"Fill in the BLANC: Human-free quality estimation of document summaries","volume-title":"Proceedings of the First Workshop on Evaluation and Comparison of NLP Systems","author":"Vasilyev","year":"2020"},{"key":"2024020223510482400_bib66","article-title":"Benchmarking generalization via in-context instructions on 1, 600+ language tasks","author":"Wang","year":"2022","journal-title":"arXiv preprint arXiv:2204.07705"},{"key":"2024020223510482400_bib67","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5283","article-title":"Fair work: Crowd work minimum wage with one line of code","volume-title":"AAAI Conference on Human Computation & Crowdsourcing","author":"Whiting","year":"2019"},{"key":"2024020223510482400_bib68","article-title":"Recursively summarizing books with human feedback","author":"Jeff","year":"2021","journal-title":"arXiv preprint arXiv:2109.10862"},{"key":"2024020223510482400_bib69","article-title":"Bartscore: Evaluating generated text as text generation","author":"Yuan","year":"2021","journal-title":"ArXiv"},{"key":"2024020223510482400_bib70","article-title":"Pegasus: Pre-training with extracted gap-sentences for abstractive summarization","volume-title":"ICML","author":"Zhang","year":"2020"},{"key":"2024020223510482400_bib71","article-title":"Opt: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"ArXiv"},{"key":"2024020223510482400_bib72","article-title":"Bertscore: Evaluating text generation with bert","volume-title":"International Conference on Learning Representations","author":"Zhang","year":"2020"},{"key":"2024020223510482400_bib73","article-title":"Fine-tuning language models from human preferences","author":"Ziegler","year":"2019","journal-title":"arXiv preprint arXiv:1909.08593"}],"container-title":["Transactions of the Association for Computational Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00632\/2325685\/tacl_a_00632.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00632\/2325685\/tacl_a_00632.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,2]],"date-time":"2024-02-02T23:51:36Z","timestamp":1706917896000},"score":1,"resource":{"primary":{"URL":"https:\/\/direct.mit.edu\/tacl\/article\/doi\/10.1162\/tacl_a_00632\/119276\/Benchmarking-Large-Language-Models-for-News"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":73,"URL":"https:\/\/doi.org\/10.1162\/tacl_a_00632","relation":{},"ISSN":["2307-387X"],"issn-type":[{"value":"2307-387X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024]]},"published":{"date-parts":[[2024]]}}}