{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T11:52:25Z","timestamp":1783597945691,"version":"3.55.0"},"reference-count":87,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T00:00:00Z","timestamp":1725062400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T00:00:00Z","timestamp":1725062400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1007\/s10115-024-02217-0","type":"journal-article","created":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T14:01:59Z","timestamp":1725112919000},"page":"7717-7738","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":18,"title":["Evaluation metrics on text summarization: comprehensive survey"],"prefix":"10.1007","volume":"66","author":[{"given":"Ensieh","family":"Davoodijam","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mohsen","family":"Alambardar\u00a0Meybodi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,8,31]]},"reference":[{"key":"2217_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-011-9216-z","volume":"37","author":"E Lloret","year":"2012","unstructured":"Lloret E, Palomar M (2012) Text summarisation in progress: a literature review. Artif Intell Rev 37:1\u201341","journal-title":"Artif Intell Rev"},{"issue":"4","key":"2217_CR2","first-page":"1029","volume":"34","author":"AP Widyassari","year":"2022","unstructured":"Widyassari AP, Rustad S, Shidik GF, Noersasongko E, Syukur A, Affandy A et al (2022) Review of automatic text summarization techniques & methods. J King Saud Univ Comput Inf Sci 34(4):1029\u20131046","journal-title":"J King Saud Univ Comput Inf Sci"},{"key":"2217_CR3","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2020.113679","volume":"165","author":"S El-Kassas Wafaa","year":"2021","unstructured":"El-Kassas Wafaa S, Salama Cherif R, Rafea Ahmed A, Mohamed Hoda K (2021) Automatic text summarization: a comprehensive survey. Expert Syst Appl 165:113679","journal-title":"Expert Syst Appl"},{"issue":"1","key":"2217_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-016-9475-9","volume":"47","author":"M Gambhir","year":"2017","unstructured":"Gambhir M, Gupta V (2017) Recent automatic text summarization techniques: a survey. Artif Intell Rev 47(1):1\u201366","journal-title":"Artif Intell Rev"},{"key":"2217_CR5","doi-asserted-by":"crossref","unstructured":"Radev DR, Blair-Goldensohn S, Zhang Z (2001) Experiments in single and multidocument summarization using mead. In: First document understanding conference, pp 1\u20137","DOI":"10.3115\/1072133.1072208"},{"key":"2217_CR6","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1016\/j.knosys.2016.01.030","volume":"99","author":"J-P Qiang","year":"2016","unstructured":"Qiang J-P, Chen P, Ding W, Xie F, Xindong W (2016) Multi-document summarization using closed patterns. Knowl-Based Syst 99:28\u201338","journal-title":"Knowl-Based Syst"},{"key":"2217_CR7","doi-asserted-by":"publisher","first-page":"385","DOI":"10.1016\/j.eswa.2017.05.075","volume":"86","author":"A John","year":"2017","unstructured":"John A, Premjith PS, Wilscy M (2017) Extractive multi-document summarization using population-based multicriteria optimization. Expert Syst Appl 86:385\u2013397","journal-title":"Expert Syst Appl"},{"key":"2217_CR8","doi-asserted-by":"crossref","unstructured":"Widjanarko A, Kusumaningrum R, Surarso B (2018) Multi document summarization for the Indonesian language based on latent Dirichlet allocation and significance sentence. In: 2018 International conference on information and communications technology (ICOIACT). IEEE, pp 520\u2013524","DOI":"10.1109\/ICOIACT.2018.8350668"},{"issue":"1","key":"2217_CR9","first-page":"64","volume":"59","author":"A Khan","year":"2014","unstructured":"Khan A, Salim N (2014) A review on abstractive summarization methods. J Theor Appl Inf Technol 59(1):64\u201372","journal-title":"J Theor Appl Inf Technol"},{"key":"2217_CR10","doi-asserted-by":"crossref","unstructured":"Deutsch D, Dror R, Roth D (2022) Re-examining system-level correlations of automatic summarization evaluation metrics. arXiv preprint arXiv:2204.10216","DOI":"10.18653\/v1\/2022.naacl-main.442"},{"key":"2217_CR11","unstructured":"Lin J, Demner-Fushman D (2005) Evaluating summaries and answers: two sides of the same coin? In: Proceedings of the ACL workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization, pp 41\u201348"},{"key":"2217_CR12","unstructured":"Lin C-Y (2004) Rouge: a package for automatic evaluation of summaries. In: Text summarization branches out, pp 74\u201381"},{"key":"2217_CR13","doi-asserted-by":"crossref","unstructured":"Papineni K, Roukos S, Ward T, Zhu W-J (2002) Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting of the association for computational linguistics, pp 311\u2013318","DOI":"10.3115\/1073083.1073135"},{"key":"2217_CR14","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1007\/s10579-017-9399-2","volume":"52","author":"E Lloret","year":"2018","unstructured":"Lloret E, Plaza L, Aker A (2018) The challenging task of summary evaluation: an overview. Lang Resour Eval 52:101\u2013148","journal-title":"Lang Resour Eval"},{"key":"2217_CR15","unstructured":"Jones KS, Galliers JR (1995) Evaluating natural language processing systems: an analysis and review. Lecture Notes in Artificial Intelligence. Springer"},{"key":"2217_CR16","unstructured":"Nenkova A, Passonneau RJ (2004) Evaluating content selection in summarization: the pyramid method. In: Proceedings of the human language technology conference of the North American chapter of the association for computational linguistics: Hlt-naacl 2004, pp 145\u2013152"},{"issue":"2","key":"2217_CR17","doi-asserted-by":"publisher","first-page":"4-es","DOI":"10.1145\/1233912.1233913","volume":"4","author":"A Nenkova","year":"2007","unstructured":"Nenkova A, Passonneau R, McKeown K (2007) The pyramid method: incorporating human content selection variation in summarization evaluation. ACM Trans Speech Lang Process 4(2):4-es","journal-title":"ACM Trans Speech Lang Process"},{"key":"2217_CR18","doi-asserted-by":"crossref","unstructured":"Shapira O, Gabay D, Gao Y, Ronen H, Pasunuru R, Bansal M, Amsterdamer Y, Dagan I (2019) Crowdsourcing lightweight pyramids for manual summary evaluation. In: Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, volume 1 (Long and Short Papers), pp 682\u2013687","DOI":"10.18653\/v1\/N19-1072"},{"key":"2217_CR19","doi-asserted-by":"crossref","unstructured":"Yang Q, Passonneau R, De\u00a0Melo G (2016) Peak: pyramid evaluation via automated knowledge extraction. In: Proceedings of the AAAI conference on artificial intelligence, vol\u00a030","DOI":"10.1609\/aaai.v30i1.10336"},{"key":"2217_CR20","unstructured":"Weisstein EW (2011) Hungarian maximum matching algorithm. https:\/\/mathworld.wolfram.com\/"},{"key":"2217_CR21","doi-asserted-by":"crossref","unstructured":"Gao Y, Sun C, Passonneau RJ (2019) Automated pyramid summarization evaluation. In: Proceedings of the 23rd conference on computational natural language learning (CoNLL)","DOI":"10.18653\/v1\/K19-1038"},{"issue":"13","key":"2217_CR22","doi-asserted-by":"publisher","first-page":"2756","DOI":"10.1016\/j.ins.2007.01.034","volume":"177","author":"S Zhang","year":"2007","unstructured":"Zhang S, Zhang J, Zhang C (2007) Edua: an efficient algorithm for dynamic database mining. Inf Sci 177(13):2756\u20132767","journal-title":"Inf Sci"},{"key":"2217_CR23","doi-asserted-by":"crossref","unstructured":"Ng J-P, Abrecht V (2015) Better summarization evaluation with word embeddings for rouge. arXiv preprint arXiv:1508.06034","DOI":"10.18653\/v1\/D15-1222"},{"key":"2217_CR24","doi-asserted-by":"crossref","unstructured":"Wang A, Cho K, Lewis M (2020) Asking and answering questions to evaluate the factual consistency of summaries. In: Proceedings of the 58th annual meeting of the association for computational linguistics, pp 5008\u20135020","DOI":"10.18653\/v1\/2020.acl-main.450"},{"key":"2217_CR25","unstructured":"Hovy EH, Lin C-Y, Zhou L, Fukumoto J (2006) Automated summarization evaluation with basic elements. In: LREC, vol\u00a06, pp 604\u2013611"},{"key":"2217_CR26","unstructured":"Tratz S, Hovy E (2008) Bewte: basic elements with transformations for evaluation. In: TAC 2008 workshop"},{"issue":"3","key":"2217_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1410358.1410359","volume":"5","author":"G Giannakopoulos","year":"2008","unstructured":"Giannakopoulos G, Karkaletsis V, Vouros G, Stamatopoulos P (2008) Summarization system evaluation revisited: N-gram graphs. ACM Trans Speech Lang Process 5(3):1\u201339","journal-title":"ACM Trans Speech Lang Process"},{"key":"2217_CR28","unstructured":"Giannakopoulos G, Karkaletsis V (2010) Summarization system evaluation variations based on n-gram graphs. In: TAC, Citeseer"},{"key":"2217_CR29","doi-asserted-by":"crossref","unstructured":"Giannakopoulos G, Karkaletsis V (2013) Summary evaluation: together we stand npower-ed. In: International conference on intelligent text processing and computational linguistics. Springer, pp 436\u2013450","DOI":"10.1007\/978-3-642-37256-8_36"},{"issue":"1","key":"2217_CR30","doi-asserted-by":"publisher","first-page":"e0000417","DOI":"10.1371\/journal.pdig.0000417","volume":"3","author":"J Gallifant","year":"2024","unstructured":"Gallifant J, Fiske A, Levites Strekalova YA, Osorio-Valencia JS, Parke R, Mwavu R, Martinez N, Gichoya JW, Ghassemi M, Demner-Fushman D et al (2024) Peer review of gpt-4 technical report and systems card. PLoS Digit Health 3(1):e0000417","journal-title":"PLoS Digit Health"},{"key":"2217_CR31","doi-asserted-by":"crossref","unstructured":"Fu J, Ng S-K, Jiang Z, Liu P (2023) Gptscore: evaluate as you desire. arXiv preprint arXiv:2302.04166","DOI":"10.18653\/v1\/2024.naacl-long.365"},{"key":"2217_CR32","doi-asserted-by":"crossref","unstructured":"Liu Y, Iter D, Xu Y, Wang S, Xu R, Zhu C (2023) Gpteval: Nlg evaluation using gpt-4 with better human alignment. arXiv preprint arXiv:2303.16634","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"2217_CR33","unstructured":"Devlin J, Chang M-W, Lee K, Toutanova K (2018) Bert: pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805"},{"key":"2217_CR34","unstructured":"Zhang T, Kishore V, Wu F, Weinberger KQ, Artzi Y (2019) Bertscore: evaluating text generation with bert. arXiv preprint arXiv:1904.09675"},{"key":"2217_CR35","unstructured":"Liu Y (2019) Fine-tune bert for extractive summarization. arXiv preprint arXiv:1903.10318"},{"key":"2217_CR36","doi-asserted-by":"crossref","unstructured":"Zhao W, Peyrard M, Liu F, Gao Y, Meyer CM, Eger S (2019) Moverscore: Text generation evaluating with contextualized embeddings and earth mover distance. arXiv preprint arXiv:1909.02622","DOI":"10.18653\/v1\/D19-1053"},{"key":"2217_CR37","doi-asserted-by":"crossref","unstructured":"Colombo PJA, Clavel C, Piantanida P (2022) Infolm: a new metric to evaluate summarization & data2text generation. In: Proceedings of the AAAI conference on artificial intelligence, vol\u00a036, pp 10554\u201310562","DOI":"10.1609\/aaai.v36i10.21299"},{"key":"2217_CR38","doi-asserted-by":"crossref","unstructured":"Deutsch D, Dror R, Roth D (2022) On the limitations of reference-free evaluations of generated text. arXiv preprint arXiv:2210.12563","DOI":"10.18653\/v1\/2022.emnlp-main.753"},{"key":"2217_CR39","unstructured":"Narayan S, Vlachos A, et\u00a0al (2019) Highres: Highlight-based reference-less evaluation of summarization. arXiv preprint arXiv:1906.01361"},{"key":"2217_CR40","doi-asserted-by":"crossref","unstructured":"Vasilyev O, Dharnidharka V, Bohannon J (2020) Fill in the blanc: human-free quality estimation of document summaries. arXiv preprint arXiv:2002.09836","DOI":"10.18653\/v1\/2020.eval4nlp-1.2"},{"issue":"4","key":"2217_CR41","doi-asserted-by":"crossref","first-page":"415","DOI":"10.1177\/107769905303000401","volume":"30","author":"WL Taylor","year":"1953","unstructured":"Taylor WL (1953) \u201cCloze procedure\u2019\u2019: a new tool for measuring readability. Journal Q 30(4):415\u2013433","journal-title":"Journal Q"},{"key":"2217_CR42","doi-asserted-by":"crossref","unstructured":"Gao Y, Zhao W, Eger S (2020) Supert: Towards new frontiers in unsupervised evaluation metrics for multi-document summarization. arXiv preprint arXiv:2005.03724","DOI":"10.18653\/v1\/2020.acl-main.124"},{"key":"2217_CR43","unstructured":"Turian J, Shen L, Melamed ID (2003) Evaluation of machine translation and its evaluation. In: Proceedings of machine translation summit IX: papers"},{"issue":"4","key":"2217_CR44","doi-asserted-by":"publisher","first-page":"1006","DOI":"10.3390\/math11041006","volume":"11","author":"S Lee","year":"2023","unstructured":"Lee S, Lee J, Moon H, Park C, Seo J, Eo S, Koo S, Lim H (2023) A survey on evaluation metrics for machine translation. Mathematics 11(4):1006","journal-title":"Mathematics"},{"key":"2217_CR45","doi-asserted-by":"crossref","unstructured":"Popovi\u0107 M (2015) chrf: character n-gram f-score for automatic mt evaluation. In: Proceedings of the tenth workshop on statistical machine translation, pp 392\u2013395","DOI":"10.18653\/v1\/W15-3049"},{"key":"2217_CR46","doi-asserted-by":"crossref","unstructured":"Sellam T, Das D, Parikh AP (2020) Bleurt: learning robust metrics for text generation. arXiv preprint arXiv:2004.04696","DOI":"10.18653\/v1\/2020.acl-main.704"},{"key":"2217_CR47","unstructured":"Banerjee S, Lavie A (2005) Meteor: an automatic metric for mt evaluation with improved correlation with human judgments. In: Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization, pp 65\u201372"},{"key":"2217_CR48","unstructured":"Denkowski M, Lavie A (2010) Meteor-next and the meteor paraphrase tables: improved evaluation support for five target languages. In: Proceedings of the joint fifth workshop on statistical machine translation and MetricsMATR, pp 339\u2013342"},{"key":"2217_CR49","doi-asserted-by":"crossref","unstructured":"Agarwal A, Lavie A (2008) Meteor, m-bleu and m-ter: evaluation metrics for high-correlation with human rankings of machine translation output. In: Proceedings of the third workshop on statistical machine translation, pp 115\u2013118","DOI":"10.3115\/1626394.1626406"},{"key":"2217_CR50","unstructured":"Denkowski M, Lavie A (2010) Extending the meteor machine translation evaluation metric to the phrase level. In: Human language technologies: the 2010 annual conference of the North American chapter of the association for computational linguistics, pp 250\u2013253"},{"key":"2217_CR51","unstructured":"Denkowski M, Lavie A (2011) Meteor 1.3: automatic metric for reliable optimization and evaluation of machine translation systems. In: Proceedings of the sixth workshop on statistical machine translation, pp 85\u201391"},{"key":"2217_CR52","doi-asserted-by":"crossref","unstructured":"Denkowski M, Lavie A (2014) Meteor universal: Language specific translation evaluation for any target language. In: Proceedings of the ninth workshop on statistical machine translation, pp 376\u2013380","DOI":"10.3115\/v1\/W14-3348"},{"key":"2217_CR53","doi-asserted-by":"crossref","unstructured":"Thompson B, Post M (2020) Automatic machine translation evaluation in many languages via zero-shot paraphrasing. arXiv preprint arXiv:2004.14564","DOI":"10.18653\/v1\/2020.emnlp-main.8"},{"key":"2217_CR54","unstructured":"Rei R, Farinha AC, Zerva C, van Stigt D, Stewart C, Ramos P, Glushkova T, Martins AFT, Lavie A (2021) Are references really needed? unbabel-ist 2021 submission for the metrics shared task. In: Proceedings of the sixth conference on machine translation, pp 1030\u20131040"},{"key":"2217_CR55","doi-asserted-by":"crossref","unstructured":"Rei R, Stewart C, Farinha AC, Lavie A (2020) Comet: a neural framework for mt evaluation. arXiv preprint[SPACE]arXiv:2009.09025","DOI":"10.18653\/v1\/2020.emnlp-main.213"},{"key":"2217_CR56","doi-asserted-by":"crossref","unstructured":"Conneau A, Khandelwal K, Goyal N, Chaudhary V, Wenzek, G Guzm\u00e1n F, Grave E, Ott M, Zettlemoyer L, Stoyanov V (2019) Unsupervised cross-lingual representation learning at scale. arXiv preprint arXiv:1911.02116","DOI":"10.18653\/v1\/2020.acl-main.747"},{"key":"2217_CR57","doi-asserted-by":"crossref","unstructured":"Xenouleas S, Malakasiotis P, Apidianaki M, Androutsopoulos I (2019) Sumqe: a bert-based summary quality estimation model. arXiv preprint arXiv:1909.00578","DOI":"10.18653\/v1\/D19-1618"},{"key":"2217_CR58","first-page":"27263","volume":"34","author":"W Yuan","year":"2021","unstructured":"Yuan W, Neubig G, Liu P (2021) Bartscore: Evaluating generated text as text generation. Adv Neural Inf Process Syst 34:27263\u201327277","journal-title":"Adv Neural Inf Process Syst"},{"key":"2217_CR59","doi-asserted-by":"crossref","unstructured":"Lewis M, Liu Y, Goyal N, Ghazvininejad M, Mohamed A, Levy O, Stoyanov V, Zettlemoyer L (2019) Bart: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension. arXiv preprint arXiv:1910.13461","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"2217_CR60","doi-asserted-by":"crossref","unstructured":"Shimanaka H, Kajiwara T, Komachi M (2018) Ruse: Regressor using sentence embeddings for automatic machine translation evaluation. In: Proceedings of the third conference on machine translation: shared task papers, pp 751\u2013758","DOI":"10.18653\/v1\/W18-6456"},{"key":"2217_CR61","doi-asserted-by":"crossref","unstructured":"Lo C (2019) Yisi-a unified semantic mt quality evaluation and estimation metric for languages with different levels of available resources. In: Proceedings of the fourth conference on machine translation (volume 2: shared task papers, Day 1), pp 507\u2013513","DOI":"10.18653\/v1\/W19-5358"},{"key":"2217_CR62","doi-asserted-by":"publisher","first-page":"774","DOI":"10.1162\/tacl_a_00397","volume":"9","author":"D Deutsch","year":"2021","unstructured":"Deutsch D, Bedrax-Weiss T, Roth D (2021) Towards question-answering as an automatic metric for evaluating the content quality of a summary. Trans Assoc Comput Linguist 9:774\u2013789","journal-title":"Trans Assoc Comput Linguist"},{"key":"2217_CR63","doi-asserted-by":"crossref","unstructured":"Eyal M, Baumel T, Elhadad M (2019) Question answering as an automatic evaluation metric for news article summarization. arXiv preprint arXiv:1906.00318","DOI":"10.18653\/v1\/N19-1395"},{"key":"2217_CR64","doi-asserted-by":"crossref","unstructured":"Scialom T, Lamprier S, Piwowarski B, Staiano J (2019) Answers unite! unsupervised metrics for reinforced summarization models. arXiv preprint[SPACE]arXiv:1909.01610","DOI":"10.18653\/v1\/D19-1320"},{"key":"2217_CR65","doi-asserted-by":"crossref","unstructured":"Durmus E, He H, Diab M (2020) Feqa: a question answering evaluation framework for faithfulness assessment in abstractive summarization. arXiv preprint arXiv:2005.03754","DOI":"10.18653\/v1\/2020.acl-main.454"},{"key":"2217_CR66","doi-asserted-by":"crossref","unstructured":"Nema P, Khapra MM (2018) Towards a better metric for evaluating question generation systems. arXiv preprint arXiv:1808.10192","DOI":"10.18653\/v1\/D18-1429"},{"key":"2217_CR67","doi-asserted-by":"crossref","unstructured":"Scialom T, Dray P-A, Gallinari P, Lamprier S, Piwowarski B, Staiano J, Wang A (2021) Questeval: Summarization asks for fact-based evaluation. arXiv preprint arXiv:2103.12693","DOI":"10.18653\/v1\/2021.emnlp-main.529"},{"key":"2217_CR68","doi-asserted-by":"crossref","unstructured":"Mesgar M, Ribeiro LFR, Gurevych I (2021) A neural graph-based local coherence model. In: Findings of the association for computational linguistics: EMNLP 2021, pp 2316\u20132321","DOI":"10.18653\/v1\/2021.findings-emnlp.199"},{"key":"2217_CR69","unstructured":"Mohiuddin T, Joty S, Nguyen DT (2018) Coherence modeling of asynchronous conversations: a neural entity grid approach. arXiv preprint arXiv:1805.02275"},{"issue":"1","key":"2217_CR70","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1162\/coli.2008.34.1.1","volume":"34","author":"R Barzilay","year":"2008","unstructured":"Barzilay R, Lapata M (2008) Modeling local coherence: an entity-based approach. Comput Linguist 34(1):1\u201334","journal-title":"Comput Linguist"},{"key":"2217_CR71","doi-asserted-by":"crossref","unstructured":"Nguyen DT, Joty S (2017) A neural local coherence model. In: Proceedings of the 55th annual meeting of the association for computational linguistics (volume 1: long papers), pp 1320\u20131330","DOI":"10.18653\/v1\/P17-1121"},{"key":"2217_CR72","unstructured":"Elsner M, Charniak E (2011) Extending the entity grid with entity-specific features. In: Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies, pp 125\u2013129"},{"key":"2217_CR73","unstructured":"Guinaudeau C, Strube M (2013) Graph-based local coherence modeling. In: Proceedings of the 51st annual meeting of the association for computational linguistics (volume 1: long papers), pp 93\u2013103"},{"key":"2217_CR74","unstructured":"Cohan A, Goharian N (2016) Revisiting summarization evaluation for scientific articles. arXiv preprint arXiv:1604.00400"},{"key":"2217_CR75","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1016\/j.datak.2017.09.001","volume":"113","author":"LA Cabrera-Diego","year":"2018","unstructured":"Cabrera-Diego LA, Torres-Moreno J-M (2018) Summtriver: a new trivergent model to evaluate summaries automatically without human references. Data Knowl Eng 113:184\u2013197","journal-title":"Data Knowl Eng"},{"issue":"8","key":"2217_CR76","doi-asserted-by":"publisher","first-page":"393","DOI":"10.3390\/info13080393","volume":"13","author":"A Chaves","year":"2022","unstructured":"Chaves A, Kesiku C, Garcia-Zapirain B (2022) Automatic text summarization of biomedical text data: a systematic review. Information 13(8):393","journal-title":"Information"},{"key":"2217_CR77","unstructured":"Dang HT, Owczarzak K et\u00a0al (2008) Overview of the tac 2008 update summarization task. In: TAC"},{"key":"2217_CR78","unstructured":"Hermann KM, Kocisky T, Grefenstette E, Espeholt L, Kay W, Suleyman M, Blunsom P (2015) Teaching machines to read and comprehend. In: Advances in neural information processing systems, vol 28"},{"key":"2217_CR79","doi-asserted-by":"crossref","unstructured":"Grusky M, Naaman M, Artzi Y (2018) Newsroom: a dataset of 1.3 million summaries with diverse extractive strategies. arXiv preprint arXiv:1804.11283","DOI":"10.18653\/v1\/N18-1065"},{"issue":"1","key":"2217_CR80","doi-asserted-by":"publisher","first-page":"170","DOI":"10.1038\/s41597-023-02068-4","volume":"10","author":"A Krithara","year":"2023","unstructured":"Krithara A, Nentidis A, Bougiatiotis K, Paliouras G (2023) Bioasq-qa: a manually curated corpus for biomedical question answering. Sci Data 10(1):170","journal-title":"Sci Data"},{"issue":"1","key":"2217_CR81","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s12859-015-0564-6","volume":"16","author":"G Tsatsaronis","year":"2015","unstructured":"Tsatsaronis G, Balikas G, Malakasiotis P, Partalas I, Zschunke M, Alvers MR, Weissenborn D, Krithara A, Petridis S, Polychronopoulos D et al (2015) An overview of the bioasq large-scale biomedical semantic indexing and question answering competition. BMC Bioinform 16(1):1\u201328","journal-title":"BMC Bioinform"},{"key":"2217_CR82","unstructured":"Rankel PA, Conroy J, Dang HT, Nenkova A (2013) A decade of automatic content evaluation of news summaries: Reassessing the state of the art. In: Proceedings of the 51st annual meeting of the association for computational linguistics (volume 2: short papers), pp 131\u2013136"},{"key":"2217_CR83","doi-asserted-by":"crossref","unstructured":"Graham Y (2015) Re-evaluating automatic summarization with bleu and 192 shades of rouge. In: Proceedings of the 2015 conference on empirical methods in natural language processing, pp 128\u2013137","DOI":"10.18653\/v1\/D15-1013"},{"key":"2217_CR84","doi-asserted-by":"crossref","unstructured":"Peyrard M (2019) Studying summarization evaluation metrics in the appropriate scoring range. In: Proceedings of the 57th annual meeting of the association for computational linguistics, pp 5093\u20135100","DOI":"10.18653\/v1\/P19-1502"},{"key":"2217_CR85","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1162\/tacl_a_00373","volume":"9","author":"AR Fabbri","year":"2021","unstructured":"Fabbri AR, Kry\u015bci\u0144ski W, McCann B, Xiong C, Socher R, Radev D (2021) Summeval: re-evaluating summarization evaluation. Trans Assoc Comput Linguist 9:391\u2013409","journal-title":"Trans Assoc Comput Linguist"},{"key":"2217_CR86","doi-asserted-by":"crossref","unstructured":"Wang J, Liang Y, Meng F, Sun Z, Shi H, Li Z, Xu J, Qu J, Zhou J (2023). Is chatgpt a good nlg evaluator? A preliminary study. arXiv preprint arXiv:2303.04048","DOI":"10.18653\/v1\/2023.newsum-1.1"},{"key":"2217_CR87","doi-asserted-by":"crossref","unstructured":"Bhandari M, Gour P, Ashfaq A, Liu P, Neubig G (2020) Re-evaluating evaluation in text summarization. arXiv preprint arXiv:2010.07100","DOI":"10.18653\/v1\/2020.emnlp-main.751"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02217-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10115-024-02217-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02217-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,27]],"date-time":"2024-11-27T16:13:08Z","timestamp":1732723988000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10115-024-02217-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,31]]},"references-count":87,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["2217"],"URL":"https:\/\/doi.org\/10.1007\/s10115-024-02217-0","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,8,31]]},"assertion":[{"value":"10 August 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 March 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 August 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"31 August 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}