{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,4,12]],"date-time":"2025-04-12T09:28:06Z","timestamp":1744450086575,"version":"3.40.3"},"publisher-location":"Cham","reference-count":42,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031243394"},{"type":"electronic","value":"9783031243400"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-24340-0_40","type":"book-chapter","created":{"date-parts":[[2023,2,25]],"date-time":"2023-02-25T13:03:43Z","timestamp":1677330223000},"page":"529-545","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["EASY: Evaluation System for\u00a0Summarization"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3044-3681","authenticated-orcid":false,"given":"Marina","family":"Litvak","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4939-1415","authenticated-orcid":false,"given":"Natalia","family":"Vanetik","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4169-8690","authenticated-orcid":false,"given":"Yael","family":"Veksler","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,2,26]]},"reference":[{"key":"40_CR1","unstructured":"Abdi, A., Idris, N.: Automated summarization assessment system: quality assessment without a reference summary. In: The International Conference on Advances in Applied Science and Environmental Engineering-ASEE (2014)"},{"key":"40_CR2","first-page":"993","volume":"3","author":"DM Blei","year":"2003","unstructured":"Blei, D.M., Ng, A.Y., Jordan, M.I.: Latent Dirichlet allocation. J. Mach. Learn. Res. 3, 993\u20131022 (2003)","journal-title":"J. Mach. Learn. Res."},{"key":"40_CR3","unstructured":"Cohan, A., Goharian, N.: Revisiting summarization evaluation for scientific articles. arXiv preprint arXiv:1604.00400 (2016)"},{"key":"40_CR4","first-page":"192","volume":"4","author":"D Das","year":"2007","unstructured":"Das, D., Martins, A.F.: A survey on automatic text summarization. Lit. Surv. Lang. Stat. II Course CMU 4, 192\u2013195 (2007)","journal-title":"Lit. Surv. Lang. Stat. II Course CMU"},{"key":"40_CR5","doi-asserted-by":"crossref","unstructured":"Davis, S.T., Conroy, J.M., Schlesinger, J.D.: OCCAMS-an optimal combinatorial covering algorithm for multi-document summarization. In: 2012 IEEE 12th International Conference on Data Mining Workshops (ICDMW), pp. 454\u2013463. IEEE (2012)","DOI":"10.1109\/ICDMW.2012.50"},{"issue":"6","key":"40_CR6","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1002\/(SICI)1097-4571(199009)41:6<391::AID-ASI1>3.0.CO;2-9","volume":"41","author":"S Deerwester","year":"1990","unstructured":"Deerwester, S., Dumais, S.T., Furnas, G.W., Landauer, T.K., Harshman, R.: Indexing by latent semantic analysis. J. Am. Soc. Inf. Sci. 41(6), 391\u2013407 (1990)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"40_CR7","doi-asserted-by":"crossref","unstructured":"Donaway, R.L., Drummey, K.W., Mather, L.A.: A comparison of rankings produced by summarization evaluation measures. In: Proceedings of the 2000 NAACL-ANLP Workshop on Automatic summarization, pp. 69\u201378. Association for Computational Linguistics (2000)","DOI":"10.3115\/1117575.1117583"},{"issue":"1","key":"40_CR8","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-016-9475-9","volume":"47","author":"M Gambhir","year":"2017","unstructured":"Gambhir, M., Gupta, V.: Recent automatic text summarization techniques: a survey. Artif. Intell. Rev. 47(1), 1\u201366 (2017)","journal-title":"Artif. Intell. Rev."},{"key":"40_CR9","unstructured":"Giannakopoulos, G., Karkaletsis, V.: Autosummeng and memog in evaluating guided summaries. In: Proceedings of Text Analysis Conference (2011)"},{"key":"40_CR10","doi-asserted-by":"crossref","unstructured":"Giannakopoulos, G., et al.: Multiling 2015: multilingual summarization of single and multi-documents, on-line fora, and call-center conversations. In: Proceedings of the 16th Annual Meeting of the Special Interest Group on Discourse and Dialogue, pp. 270\u2013274 (2015)","DOI":"10.18653\/v1\/W15-4638"},{"issue":"3","key":"40_CR11","first-page":"258","volume":"2","author":"V Gupta","year":"2010","unstructured":"Gupta, V., Lehal, G.S.: A survey of text summarization extractive techniques. J. Emerg. Technol. Web Intell. 2(3), 258\u2013268 (2010)","journal-title":"J. Emerg. Technol. Web Intell."},{"key":"40_CR12","unstructured":"Hovy, E., Lin, C.Y., Zhou, L., Fukumoto, J.: Automated summarization evaluation with basic elements. In: Proceedings of the Fifth Conference on Language Resources and Evaluation (LREC 2006), pp. 604\u2013611. Citeseer (2006)"},{"key":"40_CR13","unstructured":"Jing, H., Barzilay, R., McKeown, K., Elhadad, M.: Summarization evaluation methods: experiments and analysis. In: AAAI Symposium on Intelligent Summarization, Palo Alto, CA, pp. 51\u201359 (1998)"},{"key":"40_CR14","doi-asserted-by":"crossref","unstructured":"Jing, H., McKeown, K.R.: The decomposition of human-written summary sentences. In: Proceedings of the 22nd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 129\u2013136. ACM (1999)","DOI":"10.1145\/312624.312666"},{"key":"40_CR15","doi-asserted-by":"publisher","DOI":"10.1007\/BFb0027470","volume-title":"Evaluating Natural Language Processing Systems: An Analysis and Review","author":"KS Jones","year":"1995","unstructured":"Jones, K.S., Galliers, J.R.: Evaluating Natural Language Processing Systems: An Analysis and Review, vol. 1083. Springer, Heidelberg (1995). https:\/\/doi.org\/10.1007\/BFb0027470"},{"issue":"3","key":"40_CR16","doi-asserted-by":"publisher","first-page":"499","DOI":"10.1137\/S0036144501387141","volume":"43","author":"DR Karger","year":"2001","unstructured":"Karger, D.R.: A randomized fully polynomial time approximation scheme for the all-terminal network reliability problem. SIAM Rev. 43(3), 499\u2013522 (2001)","journal-title":"SIAM Rev."},{"issue":"6","key":"40_CR17","first-page":"53","volume":"1","author":"N Kasture","year":"2014","unstructured":"Kasture, N., Yargal, N., Singh, N.N., Kulkarni, N., Mathur, V.: A survey on methods of abstractive text summarization. Int. J. Res. Merg. Sci. Technol 1(6), 53\u201357 (2014)","journal-title":"Int. J. Res. Merg. Sci. Technol"},{"issue":"1","key":"40_CR18","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1016\/S0020-0190(99)00031-9","volume":"70","author":"S Khuller","year":"1999","unstructured":"Khuller, S., Moss, A., Naor, J.S.: The budgeted maximum coverage problem. Inf. Process. Lett. 70(1), 39\u201345 (1999)","journal-title":"Inf. Process. Lett."},{"key":"40_CR19","doi-asserted-by":"crossref","unstructured":"Kupiec, J., Pedersen, J., Chen, F.: A trainable document summarizer. In: Proceedings of the 18th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 68\u201373. ACM (1995)","DOI":"10.1145\/215206.215333"},{"key":"40_CR20","unstructured":"Kusner, M., Sun, Y., Kolkin, N., Weinberger, K.: From word embeddings to document distances. In: International Conference on Machine Learning, pp. 957\u2013966 (2015)"},{"key":"40_CR21","unstructured":"Le, Q., Mikolov, T.: Distributed representations of sentences and documents. In: International Conference on Machine Learning, pp. 1188\u20131196 (2014)"},{"key":"40_CR22","unstructured":"Lin, C.Y.: ROUGE: a package for automatic evaluation of summaries. In: Proceedings of the Workshop on Text Summarization Branches Out (WAS 2004), pp. 25\u201326 (2004)"},{"key":"40_CR23","doi-asserted-by":"crossref","unstructured":"Lloret, E., Vodolazova, T., Moreda, P., Mu\u00f1oz, R., Palomar, M.: Are better summaries also easier to understand? analyzing text complexity in automatic summarization. In: Litvak, M., Vanetik, N. (eds.) Multilingual Text Analysis: Challenges, Models, and Approaches, chap. 10. World Scientific (2019)","DOI":"10.1142\/9789813274884_0010"},{"key":"40_CR24","doi-asserted-by":"crossref","unstructured":"Mani, I.: Summarization evaluation: an overview (2001)","DOI":"10.1075\/nlp.3"},{"issue":"1","key":"40_CR25","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1017\/S1351324901002741","volume":"8","author":"I Mani","year":"2002","unstructured":"Mani, I., Klein, G., House, D., Hirschman, L., Firmin, T., Sundheim, B.: Summac: a text summarization evaluation. Nat. Lang. Eng. 8(1), 43\u201368 (2002)","journal-title":"Nat. Lang. Eng."},{"key":"40_CR26","doi-asserted-by":"crossref","unstructured":"McDonald, R.: A study of global inference algorithms in multi-document summarization. In: Advances in Information Retrieval, pp. 557\u2013564 (2007)","DOI":"10.1007\/978-3-540-71496-5_51"},{"key":"40_CR27","volume-title":"An Empirical Study of the Optimal Presentation of Multimedia Summaries of Broadcast News","author":"A Merlino","year":"1999","unstructured":"Merlino, A., Maybury, M.: An Empirical Study of the Optimal Presentation of Multimedia Summaries of Broadcast News. MIT Press, Cambridge (1999)"},{"key":"40_CR28","unstructured":"Mihalcea, R., Tarau, P.: TextRank: bringing order into text. In: Proceedings of the 2004 Conference on Empirical Methods in Natural Language Processing (2004)"},{"key":"40_CR29","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: Advances in Neural Information Processing Systems, pp. 3111\u20133119 (2013)"},{"key":"40_CR30","doi-asserted-by":"crossref","unstructured":"Nanba, H., Okumura, M.: Producing more readable extracts by revising them. In: Proceedings of the 18th Conference on Computational Linguistics, vol. 2, pp. 1071\u20131075. Association for Computational Linguistics (2000)","DOI":"10.3115\/992730.992807"},{"key":"40_CR31","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1007\/978-1-4614-3223-4_3","volume-title":"Mining Text Data","author":"A Nenkova","year":"2012","unstructured":"Nenkova, A., McKeown, K.: A survey of text summarization techniques. In: Aggarwal, C., Zhai, C. (eds.) Mining Text Data, pp. 43\u201376. Springer, Boston (2012). https:\/\/doi.org\/10.1007\/978-1-4614-3223-4_3"},{"key":"40_CR32","doi-asserted-by":"crossref","unstructured":"Nenkova, A., McKeown, K., et al.: Automatic summarization. Found. Trends\u00ae Inf. Retrieval 5(2\u20133), 103\u2013233 (2011)","DOI":"10.1561\/1500000015"},{"key":"40_CR33","unstructured":"Nenkova, A., Passonneau, R.: Evaluating content selection in summarization: the pyramid method. In: Proceedings of the Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics: HLT-NAACL 2004 (2004)"},{"key":"40_CR34","doi-asserted-by":"crossref","unstructured":"Ng, J.P., Abrecht, V.: Better summarization evaluation with word embeddings for rouge. arXiv preprint arXiv:1508.06034 (2015)","DOI":"10.18653\/v1\/D15-1222"},{"key":"40_CR35","doi-asserted-by":"crossref","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th Annual Meeting on Association for Computational Linguistics, pp. 311\u2013318. Association for Computational Linguistics (2002)","DOI":"10.3115\/1073083.1073135"},{"key":"40_CR36","doi-asserted-by":"crossref","unstructured":"Pastra, K., Saggion, H.: Colouring summaries Bleu. In: Proceedings of the EACL 2003 Workshop on Evaluation Initiatives in Natural Language Processing: are Evaluation Methods, Metrics and Resources Reusable?, pp. 35\u201342. Association for Computational Linguistics (2003)","DOI":"10.3115\/1641396.1641402"},{"key":"40_CR37","doi-asserted-by":"crossref","unstructured":"Pittaras, N., Montanelliy, S., Giannakopoulos, G., Ferraray, A., Karkaletsis, V.: Crowdsourcing in single-document summary evaluation: the argo way. In: Litvak, M., Vanetik, N. (eds.) Multilingual Text Analysis: Challenges, Models, and Approaches, chap. 8. World Scientific (2019)","DOI":"10.1142\/9789813274884_0008"},{"key":"40_CR38","unstructured":"Salton, G., McGill, M.J.: Introduction to Modern Information Retrieval. McGraw-Hill, Inc., New York (1986)"},{"issue":"5","key":"40_CR39","first-page":"1","volume":"1","author":"Y Sasaki","year":"2007","unstructured":"Sasaki, Y., et al.: The truth of the f-measure. Teach Tutor Mater 1(5), 1\u20135 (2007)","journal-title":"Teach Tutor Mater"},{"key":"40_CR40","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"245","DOI":"10.1007\/978-3-540-30198-1_25","volume-title":"Advances in Information Systems","author":"J Steinberger","year":"2004","unstructured":"Steinberger, J., Je\u017eek, K.: Text summarization and singular value decomposition. In: Yakhno, T. (ed.) ADVIS 2004. LNCS, vol. 3261, pp. 245\u2013254. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-30198-1_25"},{"issue":"2","key":"40_CR41","first-page":"251","volume":"28","author":"J Steinberger","year":"2012","unstructured":"Steinberger, J., Je\u017eek, K.: Evaluation measures for text summarization. Comput. Inform. 28(2), 251\u2013275 (2012)","journal-title":"Comput. Inform."},{"key":"40_CR42","doi-asserted-by":"crossref","unstructured":"Well, A.D., Myers, J.L.: Research Design & Statistical Analysis. Psychology Press, London (2003)","DOI":"10.4324\/9781410607034"}],"container-title":["Lecture Notes in Computer Science","Computational Linguistics and Intelligent Text Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-24340-0_40","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,25]],"date-time":"2023-02-25T13:10:01Z","timestamp":1677330601000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-24340-0_40"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031243394","9783031243400"],"references-count":42,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-24340-0_40","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"26 February 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CICLing","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Computational Linguistics and Intelligent Text Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"La Rochelle","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 April 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 April 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"cicling2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.cicling.org\/2019\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"335","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"95","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"28% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}