{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T01:31:54Z","timestamp":1743039114906,"version":"3.40.3"},"publisher-location":"Cham","reference-count":25,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031243394"},{"type":"electronic","value":"9783031243400"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-24340-0_41","type":"book-chapter","created":{"date-parts":[[2023,2,25]],"date-time":"2023-02-25T13:03:43Z","timestamp":1677330223000},"page":"546-557","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Performance of Evaluation Methods Without Human References for Multi-document Text Summarization"],"prefix":"10.1007","author":[{"given":"Alexis Carriola","family":"Careaga","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yulia","family":"Ledeneva","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jonathan Rojas","family":"Sim\u00f3n","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,2,26]]},"reference":[{"key":"41_CR1","unstructured":"Conroy, J.M., et al.: CLASSY 2009: summarization and metrics. In: Proceedings of the Text Analysis Conference (TAC 2009), pp. 1\u201312. NIST, Maryland, USA (2009)"},{"key":"41_CR2","unstructured":"Devlin, J., et al.: BERT: Pre-training of deep bidirectional transformers for language understanding. In: NAACL HLT 2019 - 2019 Conference North American Chapter Association Computational Linguist. Human Language Technology - Proceedings Conference 1, Mlm, pp. 4171\u20134186 (2019)"},{"key":"41_CR3","doi-asserted-by":"publisher","unstructured":"He, T., et al.: ROUGE-C: A fully automated evaluation method for multi-document summarization. In: 2008 IEEE International Conference Granular Computing GRC 2008, pp. 269\u2013274 (2008). https:\/\/doi.org\/10.1109\/GRC.2008.4664680","DOI":"10.1109\/GRC.2008.4664680"},{"key":"41_CR4","doi-asserted-by":"publisher","unstructured":"Jones, K.S., Galliers, J.R.: Evaluating Natural Language Processing Systems. Springer Berlin Heidelberg, Berlin, Heidelberg (2009). https:\/\/doi.org\/10.1007\/BFb0027470","DOI":"10.1007\/BFb0027470"},{"key":"41_CR5","doi-asserted-by":"crossref","unstructured":"Kendall, M.G.: A new measure of rank correlation. Source: Biometrika. 30, 12, 81\u201393 (1938)","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"41_CR6","unstructured":"Ledeneva, Y., Garc\u00eda-Hern\u00e1ndez, R.A.: Automatic generation of text summaries: challenges, proposals and experiments. Auton. Univ. State Mex. Toluca (2017)"},{"key":"41_CR7","doi-asserted-by":"publisher","unstructured":"Lin, C.-Y., Hovy, E.: Manual and automatic evaluation of summaries. In: Proceedings of the ACL-02 Workshop on Automatic Summarization. Association for Computational Linguistics, Morristown, NJ, USA, pp. 45\u201351 (2002). https:\/\/doi.org\/10.3115\/1118162.1118168","DOI":"10.3115\/1118162.1118168"},{"key":"41_CR8","unstructured":"Lin, C.Y.: Rouge: a package for automatic evaluation of summaries. In: Proceedings Work. text Summ. branches out (WAS 2004). 1, pp. 25\u201326 (2004)"},{"issue":"1","key":"41_CR9","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1007\/s10579-017-9399-2","volume":"52","author":"E Lloret","year":"2017","unstructured":"Lloret, E., Plaza, L., Aker, A.: The challenging task of summary evaluation: an overview. Lang. Resour. Eval. 52(1), 101\u2013148 (2017). https:\/\/doi.org\/10.1007\/s10579-017-9399-2","journal-title":"Lang. Resour. Eval."},{"key":"41_CR10","unstructured":"Louis, A., Nenkova, A.: Automatic Summary Evaluation without Human Models (2008)"},{"key":"41_CR11","doi-asserted-by":"publisher","unstructured":"Louis, A., Nenkova, A.: Automatically assessing machine summary content without a gold standard. Diss. Abstr. Int. B Sci. Eng. 70, 8, 4943 (2013). https:\/\/doi.org\/10.1162\/COLI","DOI":"10.1162\/COLI"},{"key":"41_CR12","doi-asserted-by":"publisher","unstructured":"Louis, A., Nenkova, A.: Automatically evaluating content selection in summarization without human models. In: EMNLP 2009 - Proc. 2009 Conf. Empir. Methods Nat. Lang. Process. A Meet. SIGDAT, a Spec. Interes. Gr. ACL, Held Conjunction with ACL-IJCNLP 2009. August, pp. 306\u2013314 (2009). https:\/\/doi.org\/10.3115\/1699510.1699550","DOI":"10.3115\/1699510.1699550"},{"key":"41_CR13","unstructured":"Mendoza, G.A.M., et al.: Detection of main ideas and production of summaries in English, Spanish, Portuguese an Russian. 60 years of research. Alfaomega Grupo Editor, S.A. de C.V. and Universidad Aut\u00f3noma del Estado de M\u00e9xico, State of Mexico, Mexico (2021)"},{"key":"41_CR14","doi-asserted-by":"publisher","unstructured":"Matias Mendoza, G.A., et al.: Evaluaci\u00f3n de las herramientas comerciales y m\u00e9todos del estado del arte para la generaci\u00f3n de res\u00famenes extractivos individuales. Res. Comput. Sci. 70, 1, 265\u2013274 (2013). https:\/\/doi.org\/10.13053\/rcs-70-1-20","DOI":"10.13053\/rcs-70-1-20"},{"key":"41_CR15","doi-asserted-by":"publisher","unstructured":"Pearson, K.: VII. Note on regression and inheritance in the case of two parents. Proc. R. Soc. Lond. 58(347\u2013352), 240\u2013242 (1895). https:\/\/doi.org\/10.1098\/rspl.1895.0041","DOI":"10.1098\/rspl.1895.0041"},{"key":"41_CR16","unstructured":"Nenkova, A., Passonneau, R.: Evaluating content selection in summarization: The pyramid method. In: Proceedings HLT-NAACL. 2004, January, pp. 145\u2013152 (2004)"},{"key":"41_CR17","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1145\/1233912.1233913","volume":"4","author":"A Nenkova","year":"2007","unstructured":"Nenkova, A., et al.: The Pyramid Method: Incorporating human content selection variation in summarization evaluation. ACM Trans. Speech Lang. Process. 4, 2 (2007). https:\/\/doi.org\/10.1145\/1233912.1233913","journal-title":"ACM Trans. Speech Lang. Process."},{"issue":"3","key":"41_CR18","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1108\/00330330610681286","volume":"40","author":"MF Porter","year":"2006","unstructured":"Porter, M.F.: An algorithm for suffix stripping. Program 40(3), 211\u2013218 (2006). https:\/\/doi.org\/10.1108\/00330330610681286","journal-title":"Program"},{"key":"41_CR19","doi-asserted-by":"publisher","unstructured":"Reimers, N., Gurevych, I.: Sentence-BERT: Sentence embeddings using siamese BERT-networks. In: EMNLP-IJCNLP 2019 - 2019 Conference Empir. Methods National Language Processing 9th International Jt. Conference National Language Processing Proceedings Conference, pp. 3982\u20133992 (2020). https:\/\/doi.org\/10.18653\/v1\/d19-1410","DOI":"10.18653\/v1\/d19-1410"},{"issue":"3","key":"41_CR20","doi-asserted-by":"publisher","first-page":"853","DOI":"10.1016\/j.eswa.2013.08.015","volume":"41","author":"G Sidorov","year":"2014","unstructured":"Sidorov, G., et al.: Syntactic N-grams as machine learning features for natural language processing. Expert Syst. Appl. 41(3), 853\u2013860 (2014). https:\/\/doi.org\/10.1016\/j.eswa.2013.08.015","journal-title":"Expert Syst. Appl."},{"key":"41_CR21","doi-asserted-by":"publisher","unstructured":"Sidorov, G.: Syntactic n-grams in Computational Linguistics. Springer International Publishing, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-14771-6","DOI":"10.1007\/978-3-030-14771-6"},{"key":"41_CR22","doi-asserted-by":"crossref","unstructured":"Spearman, C.: The Proof and Measurement of Association between Two Things. Am. J. Psychol. 15, 1, 72\u2013101 (1904)","DOI":"10.2307\/1412159"},{"issue":"2","key":"41_CR23","first-page":"251","volume":"28","author":"J Steinberger","year":"2009","unstructured":"Steinberger, J., Je\u017eek, K.: Evaluation measures for text summarization. Comput. Informatics. 28(2), 251\u2013275 (2009)","journal-title":"Comput. Informatics."},{"key":"41_CR24","unstructured":"Dang, H.T., Owczarzak, K.: Overview of TAC 2009 summarization track. In: Proceedings of the Text Analysis Conference, pp. 1\u201325. Gaithersburg, USA (2009)"},{"key":"41_CR25","unstructured":"Zhang, T., et al.: BERTScore: evaluating text generation with BERT. In: Proceedings of the International Conference on Learning Representations (ICLR 2020), pp. 1\u201343. Ethiopia (2020)"}],"container-title":["Lecture Notes in Computer Science","Computational Linguistics and Intelligent Text Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-24340-0_41","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,25]],"date-time":"2023-02-25T13:10:08Z","timestamp":1677330608000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-24340-0_41"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031243394","9783031243400"],"references-count":25,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-24340-0_41","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"26 February 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CICLing","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Computational Linguistics and Intelligent Text Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"La Rochelle","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 April 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 April 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"cicling2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.cicling.org\/2019\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"335","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"95","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"28% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}