{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T13:23:29Z","timestamp":1779197009070,"version":"3.51.4"},"reference-count":150,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,9,2]],"date-time":"2017-09-02T00:00:00Z","timestamp":1504310400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100003329","name":"Ministerio de Econom\u00eda y Competitividad","doi-asserted-by":"publisher","award":["TIN2013-47090-C3-1-P"],"award-info":[{"award-number":["TIN2013-47090-C3-1-P"]}],"id":[{"id":"10.13039\/501100003329","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003329","name":"Ministerio de Econom\u00eda y Competitividad","doi-asserted-by":"publisher","award":["TIN2015-71785-R"],"award-info":[{"award-number":["TIN2015-71785-R"]}],"id":[{"id":"10.13039\/501100003329","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003359","name":"Generalitat Valenciana","doi-asserted-by":"crossref","award":["PROMETEOII\/2014\/001"],"award-info":[{"award-number":["PROMETEOII\/2014\/001"]}],"id":[{"id":"10.13039\/501100003359","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100005367","name":"Universidad Nacional de Educaci\u00f3n a Distancia","doi-asserted-by":"crossref","award":["2014-001-UNED-PROY"],"award-info":[{"award-number":["2014-001-UNED-PROY"]}],"id":[{"id":"10.13039\/501100005367","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2018,3]]},"DOI":"10.1007\/s10579-017-9399-2","type":"journal-article","created":{"date-parts":[[2017,9,2]],"date-time":"2017-09-02T09:49:00Z","timestamp":1504345740000},"page":"101-148","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":57,"title":["The challenging task of summary evaluation: an overview"],"prefix":"10.1007","volume":"52","author":[{"given":"Elena","family":"Lloret","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Laura","family":"Plaza","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3381-0790","authenticated-orcid":false,"given":"Ahmet","family":"Aker","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2017,9,2]]},"reference":[{"key":"9399_CR1","unstructured":"Aker, A., El-Haj, M., Albakour, M.-D., & Kruschwitz, U. (2012a). Assessing crowdsourcing quality through objective tasks. In Proceedings of the eighth international conference on language resources and evaluation (LREC-2012). European Language Resources Association (ELRA), Istanbul, Turkey (pp. 1456\u201361)."},{"key":"9399_CR2","doi-asserted-by":"crossref","unstructured":"Aker, A., Fan, X., Sanderson, M., & Gaizauskas, R. (2012b). Investigating summarization techniques for geo-tagged image indexing. In Advances in information retrieval: 34th European conference on information retrieval (ECIR), Barcelona, Spain (pp. 472\u201375).","DOI":"10.1007\/978-3-642-28997-2_44"},{"key":"9399_CR3","unstructured":"Aker, A., & Gaizauskas, R. (2010). Model summaries for location-related images. In Proceedings of the 7th language resources and evaluation conference."},{"key":"9399_CR4","doi-asserted-by":"publisher","unstructured":"Alhindi, A., Kruschwitz, U., & Fox, C. (2013). A pilot study on using profile-based summarisation for interactive search assistance. In P. Serdyukov, P. Braslavski, S. Kuznetsov, J. Kamps, S. Rger, E. Agichtein, I. Segalovich & E. Yilmaz, E. (Eds.), Advances in information retrieval. Vol. 7814 of Lecture Notes in Computer Science, Springer, Berlin (pp. 672\u201375). doi: 10.1007\/978-3-642-36973-5_57 .","DOI":"10.1007\/978-3-642-36973-5%5f57"},{"key":"9399_CR5","doi-asserted-by":"crossref","unstructured":"Amigo, E., Gonzalo, J., Peinado, V., Pe\u00f1as, A., & Verdejo, F. (2004). An empirical study of information synthesis task. In Proceedings of the 42nd meeting of the association for computational linguistics (ACL\u201904), Main Volume, Barcelona, Spain (pp. 207\u201314).","DOI":"10.3115\/1218955.1218982"},{"key":"9399_CR6","doi-asserted-by":"crossref","unstructured":"Balikas, G., Krithara, A., Partalas, I., & Paliouras, G. (2015). Bioasq: A challenge on large-scale biomedical semantic indexing and question-answering. In Multimodal retrieval in the medical domain, Workshop at ECIR.","DOI":"10.1007\/978-3-319-24471-6_3"},{"key":"9399_CR7","unstructured":"Balikas, G., Partalas, I., Kosmopoulos, A., Petridis, S., Malakasiotis, P., & Pavlopoulos, I., et al. (2013). Bioasq evaluation framework specifications. Project deliverable D4.1. http:\/\/bioasq.org\/sites\/default\/files\/PublicDocuments\/BioASQ_D4.1-EvaluationFrameworkSpecification_final.pdf ."},{"key":"9399_CR8","unstructured":"Bamman, D., O\u2019Connor, B., & Smith, N. A. (2013). Learning latent personas of film characters. In ACL (1). The Association for Computer Linguistics (pp. 352\u201361)."},{"key":"9399_CR9","doi-asserted-by":"crossref","unstructured":"Banko, M., & Vanderwende, L. (2004). Using n-grams to understand the nature of summaries. In Proceedings of HLT-NAACL 2004: Short Papers. HLT-NAACL-Short \u201904. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 1\u20134). http:\/\/dl.acm.org\/citation.cfm?id=1613984.1613985 .","DOI":"10.3115\/1613984.1613985"},{"key":"9399_CR10","doi-asserted-by":"crossref","unstructured":"Barzilay, R., & Lapata, M. (2005). Modeling local coherence: An entity-based approach. In Proceedings of the 43rd annual meeting of the association for computational linguistics (ACL\u201905) (pp. 141\u201348).","DOI":"10.3115\/1219840.1219858"},{"issue":"1","key":"9399_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1162\/coli.2008.34.1.1","volume":"34","author":"R Barzilay","year":"2008","unstructured":"Barzilay, R., & Lapata, M. (2008). Modeling local coherence: An entity-based approach. Computational Linguistics, 34(1), 1\u201334.","journal-title":"Computational Linguistics"},{"key":"9399_CR12","first-page":"21","volume":"49","author":"R Berlanga Llavori","year":"2012","unstructured":"Berlanga Llavori, R., Ram\u00edrez Cruz, Y., & Gil Garc\u00eda, R. (2012). A framework for obtaining structurally complex condensed representations of document sets in the biomedical domain. Procesamiento del Lenguaje Natural, 49, 21\u20138.","journal-title":"Procesamiento del Lenguaje Natural"},{"issue":"3","key":"9399_CR13","first-page":"1","volume":"8","author":"E Branny","year":"2007","unstructured":"Branny, E. (2007). Automatic summary evaluation based on text grammars. Journal of Digital Information, 8(3), 1\u20136.","journal-title":"Journal of Digital Information"},{"key":"9399_CR14","doi-asserted-by":"crossref","unstructured":"Cabrera-Diego, L.\u00a0A., Torres-Moreno, J., & Durette, B. (2016). Evaluating multiple summaries without human models: A first experiment with a trivergent model. In Natural language processing and information systems\u201421st international conference on applications of natural language to information systems, NLDB 2016, Salford, UK, June 22\u201324, 2016, Proceedings (pp. 91\u2013101).","DOI":"10.1007\/978-3-319-41754-7_8"},{"key":"9399_CR15","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C. (2009). Fast, cheap, and creative: evaluating translation quality using Amazon\u2019s mechanical turk. In Proceedings of the 2009 conference on empirical methods in natural language processing (pp. 286\u201395).","DOI":"10.3115\/1699510.1699548"},{"key":"9399_CR16","unstructured":"Carletta, J., Ashby, S., Bourban, S., Flynn, M., Guillemot, M., Hain, T., Kadlec, J., Karaiskos, V., Kraaij, W., Kronenthal, M., Lathoud, G., Lincoln, M., Lisowska, A., McCowan, I., Post, W., Reidsma, D., & Wellner, P. (2005). The AMI meeting corpus. In L. P. J. J. Noldus, F. Grieco, L. W.\u00a0S. Loijens & P. H. Zimmerman (Eds.), Proceedings of the measuring behavior 2005 symposium on \u201cannotating and measuring meeting behavior\u201d."},{"key":"9399_CR17","unstructured":"Chen, P., & Verma, R. (2006). A query-based medical information summarization system using ontology knowledge. In Proceedings of the IEEE symposium on computer-based medical systems (pp. 37\u201342)."},{"key":"9399_CR18","unstructured":"Christensen, J., Mausam, S. S., Soderland, S., & Etzioni, O. (2013). Towards coherent multi-document summarization. In Proceedings of the 2013 conference of the North American chapter of the association for computational linguistics: Human language technologies. Association for Computational Linguistics, Atlanta, Georgia (pp. 1163\u20131173). http:\/\/www.aclweb.org\/anthology\/N13-1136 ."},{"key":"9399_CR19","first-page":"2493","volume":"12","author":"R Collobert","year":"2011","unstructured":"Collobert, R., Weston, J., Bottou, L., Karlen, M., Kavukcuoglu, K., & Kuksa, P. (2011). Natural language processing (almost) from scratch. Journal of Machine Learning Research, 12, 2493\u20132537.","journal-title":"Journal of Machine Learning Research"},{"key":"9399_CR20","doi-asserted-by":"crossref","unstructured":"Conroy, J.\u00a0M., & Dang, H.\u00a0T. (2008a). Mind the gap: Dangers of divorcing evaluations of summary content from linguistic quality. In Proceedings of the 22nd international conference on computational linguistics (Coling 2008). Coling 2008 Organizing Committee, Manchester, UK (pp. 145\u201352).","DOI":"10.3115\/1599081.1599100"},{"key":"9399_CR21","doi-asserted-by":"crossref","unstructured":"Conroy, J.\u00a0M., & Dang, H.\u00a0T. (2008b). Mind the gap: Dangers of divorcing evaluations of summary content from linguistic quality. In Proceedings of the 22nd international conference on computational linguistics\u2014Volume 1. COLING \u201908. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 145\u2013152). http:\/\/dl.acm.org\/citation.cfm?id=1599081.1599100 .","DOI":"10.3115\/1599081.1599100"},{"key":"9399_CR22","unstructured":"Conroy, J.\u00a0M., Schlesinger, J.\u00a0D., Kubina, J., Rankel, P.\u00a0A., & O\u2019Leary, D.\u00a0P. (2011). CLASSY 2011 at TAC: Guided and multi-lingual summaries and evaluation metrics. In Proceedings of the 2011 text analysis conference (TAC 2011)."},{"key":"9399_CR23","unstructured":"Conroy, J.\u00a0M., Schlesinger, J.\u00a0D., Rankel, P.\u00a0A., & O\u2019Leary, D.\u00a0P. (2010). Guiding CLASSY toward more responsive summaries. In Proceedings of the 2010 text analysis conference (TAC 2010)."},{"key":"9399_CR24","unstructured":"Dalianis, H., & Hassel, M. (2001). Development of a Swedish corpus for evaluating summarizers and other IR-tools. Technical report TRITA-NAP0112, IPLab-188, NADA, KTH."},{"key":"9399_CR25","unstructured":"Dang, H.\u00a0T. (2005). Overview of DUC 2005. In Proceedings of the document understanding conference (DUC)."},{"key":"9399_CR26","unstructured":"Dang, H.\u00a0T. (2006). Overview of DUC 2006. In Proceedings of the document understanding conference (DUC)."},{"key":"9399_CR27","doi-asserted-by":"crossref","unstructured":"Donaway, R.\u00a0L., Drummey, K.\u00a0W., & Mather, L.\u00a0A. (2000). A comparison of rankings produced by summarization evaluation measures. In Proceedings of NAACL-ANLP 2000 workshop on automatic summarization (pp. 69\u201378).","DOI":"10.3115\/1117575.1117583"},{"key":"9399_CR28","unstructured":"Dong, Z., & Dong, Q. (2003). HowNet\u2014A hybrid language and knowledge resource. In Proceedings of natural language processing and knowledge engineering conference (pp. 820\u201324)."},{"issue":"2","key":"9399_CR29","doi-asserted-by":"publisher","first-page":"264","DOI":"10.1145\/321510.321519","volume":"16","author":"HP Edmundson","year":"1969","unstructured":"Edmundson, H. P. (1969). New methods in automatic extracting. Journal of the ACM (JACM), 16(2), 264\u201385.","journal-title":"Journal of the ACM (JACM)"},{"key":"9399_CR30","unstructured":"El-Haj, M., Kruschwitz, U., & Fox, C. (2010). Using mechanical turk to create a corpus of arabic summaries. In Proceedings of the seventh conference on international language resources and evaluation, Valletta, Malta."},{"issue":"2","key":"9399_CR31","doi-asserted-by":"publisher","first-page":"179","DOI":"10.1016\/j.artmed.2004.07.018","volume":"33","author":"N Elhadad","year":"2005","unstructured":"Elhadad, N., Kan, M. Y., Klavans, J. L., & McKeown, K. R. (2005). Customization in a unified framework for summarizing medical literature. Artificial Intelligence in Medicine, 33(2), 179\u2013198. doi: 10.1016\/j.artmed.2004.07.018 .","journal-title":"Artificial Intelligence in Medicine"},{"key":"9399_CR32","doi-asserted-by":"crossref","unstructured":"Ellouze, S., Jaoua, M., & Belguith, L.\u00a0H. (2016). Automatic evaluation of a summary\u2019s linguistic quality. In Natural language processing and information systems\u201421st international conference on applications of natural language to information systems, NLDB 2016, Salford, UK, June 22\u201324, 2016, Proceedings (pp. 392\u2013400).","DOI":"10.1007\/978-3-319-41754-7_39"},{"key":"9399_CR33","doi-asserted-by":"crossref","unstructured":"Ellouze, S., Jaoua, M., & Hadrich\u00a0Belguith, L. (2017). Machine learning approach to evaluate multilingual summaries. In Proceedings of the MultiLing 2017 workshop on summarization and summary evaluation across source types and genres. Association for Computational Linguistics (pp. 47\u201354).","DOI":"10.18653\/v1\/W17-1007"},{"key":"9399_CR34","doi-asserted-by":"crossref","unstructured":"Feng, D., Besana, S., & Zajac, R. (2009). Acquiring high quality non-expert knowledge from on-demand workforce. In Proceedings of the 2009 workshop on the people\u2019s web meets NLP: Collaboratively constructed semantic resources. People\u2019s Web \u201909. Association for Computational Linguistics, Morristown, NJ, USA (pp. 51\u20136). http:\/\/portal.acm.org\/citation.cfm?id=1699765.1699773 .","DOI":"10.3115\/1699765.1699773"},{"key":"9399_CR35","unstructured":"Field, D., Pulman, S., Van\u00a0Labeke, N., Whitelock, D., & Richardson, J. (2013). Did I really mean that? Applying automatic summarisation techniques to formative feedback. In Proceedings of the international conference recent advances in natural language processing RANLP 2013. INCOMA Ltd. Shoumen, BULGARIA, Hissar, Bulgaria (pp. 277\u201384). http:\/\/www.aclweb.org\/anthology\/R13-1036 ."},{"key":"9399_CR36","doi-asserted-by":"crossref","unstructured":"Fiori, A. (2014). Innovative document summarization techniques: Revolutionizing knowledge understanding: Revolutionizing knowledge understanding. In Advances in data mining and database management: IGI Global.","DOI":"10.4018\/978-1-4666-5019-0"},{"key":"9399_CR37","doi-asserted-by":"publisher","unstructured":"Fiszman, M., Demner-Fushman, D., Kilicoglu, H., & Rindflesch, T.\u00a0C. (2009). Automatic summarization of medline citations for evidence-based medical treatment: A topic-oriented evaluation. Journal of Biomedical Informatics, 42(5), 801\u2013813. doi: 10.1016\/j.jbi.2008.10.002 .","DOI":"10.1016\/j.jbi.2008.10.002"},{"issue":"1","key":"9399_CR38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-016-9475-9","volume":"47","author":"M Gambhir","year":"2017","unstructured":"Gambhir, M., & Gupta, V. (2017). Recent automatic text summarization techniques: A survey. Artificial Intelligence Review, 47(1), 1\u201366. doi: 10.1007\/s10462-016-9475-9 .","journal-title":"Artificial Intelligence Review"},{"key":"9399_CR39","doi-asserted-by":"crossref","unstructured":"Giannakopoulos, G., Conroy, J., Kubina, J., Rankel, P.\u00a0A., Lloret, E., Steinberger, J., Litvak, M., & Favre, B. (2017). Multiling 2017 overview. In Proceedings of the MultiLing 2017 workshop on summarization and summary evaluation across source types and genres. Association for Computational Linguistics, Valencia, Spain (pp. 1\u20136). http:\/\/www.aclweb.org\/anthology\/W17-1001 .","DOI":"10.18653\/v1\/W17-1001"},{"key":"9399_CR40","unstructured":"Giannakopoulos, G., & Karkaletsis, V. (2011a). AutoSummENG and MeMoG in evaluating guided summaries. In Proceedings of the 2011 text analysis conference (TAC 2011)."},{"key":"9399_CR41","unstructured":"Giannakopoulos, G., & Karkaletsis, V. (2011b). Autosummeng and memog in evaluating guided summaries. In Proceedings of the text analysis conference (TAC 2011), Gaithersburg, Maryland, USA."},{"key":"9399_CR42","unstructured":"Giannakopoulos, G., & Karkaletsis, V. (2013). Together we stand npower-ed. In Proceedings of CICLing 2013, Karlovasi, Samos, Greece."},{"issue":"3","key":"9399_CR43","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1410358.1410359","volume":"5","author":"G Giannakopoulos","year":"2008","unstructured":"Giannakopoulos, G., Karkaletsis, V., Vouros, G., & Stamatopoulos, P. (2008). Summarization system evaluation revisited: N-gram graphs. ACM Transactions on Speech and Language Processing, 5(3), 1\u201339.","journal-title":"ACM Transactions on Speech and Language Processing"},{"key":"9399_CR44","unstructured":"Gillick, D., & Liu, Y. (2010). Non-expert evaluation of summarization systems is Risky. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk."},{"issue":"2","key":"9399_CR45","first-page":"203","volume":"21","author":"BJ Grosz","year":"1995","unstructured":"Grosz, B. J., Weinstein, S., & Joshi, A. K. (1995). Centering: A framework for modeling the local coherence of discourse. Computational Linguistics, 21(2), 203\u201325.","journal-title":"Computational Linguistics"},{"key":"9399_CR46","unstructured":"Hand, T. (1997). A proposal for task-based evaluation of text summarization systems. In Proceedings of the association for computational linguistics conference, Madrid, Spain (pp. 31\u201338)."},{"key":"9399_CR47","unstructured":"Harnly, A., Nenkova, A., Passonneau, R.\u00a0J., & Rambow, O. (2015). Automatation of summary evaluation by the pyramid method. In Proceedings of the international conference recent advances in natural language processing (RANLP), Borovets, Bulgaria (pp. 226\u2013232)."},{"key":"9399_CR48","unstructured":"Hasler, L. (2008). Centering theory for evaluation of coherence in computer-aided summaries. In Proceedings of the sixth international conference on language resources and evaluation."},{"key":"9399_CR49","unstructured":"Hasler, L., Or\u0103san, C., & Mitkov, R. (2003). Building better corpora for summarization. In Proceedings of corpus linguistics 2003, Lancaster, UK (pp. 309\u201319)."},{"key":"9399_CR50","unstructured":"Hassel, M. (2004). Evaluation of automatic text summarization: A practical implementation."},{"key":"9399_CR51","unstructured":"He, T., Chen, J., Ma, L., Gui, Z., Li, F., Shao, W., & Wang, Q. (2008). ROUGE-C: A fully automated evaluation method for multi-document summarization, Granular Computing, 2008. GrC 2008. In IEEE international conference on (pp. 269\u201374)."},{"key":"9399_CR52","unstructured":"Hong, K., Conroy, J., Favre, B., Kulesza, A., Lin, H., & Nenkova, A. (2014). A repository of state of the art and competitive baseline summaries for generic news summarization. In N. C.\u00a0C. Chair, K. Choukri, T. Declerck, H. Loftsson, B. Maegaard, J. Mariani, A. Moreno, J. Odijk, & S. Piperidis (Eds.), Proceedings of the ninth international conference on language resources and evaluation (LREC\u201914). European Language Resources Association (ELRA), Reykjavik, Iceland."},{"key":"9399_CR53","unstructured":"Hovy, E. (2005). The Oxford handbook of computational linguistics. Oxford University Press, Ch. Text Summarization (pp. 583\u201398)."},{"key":"9399_CR54","unstructured":"Hovy, E., Lin, C.-Y., Zhou, L., & Fukumoto, J. (2006). Automated summarization evaluation with basic elements. In Proceedings of the 5th international conference on language resources and evaluation."},{"key":"9399_CR55","doi-asserted-by":"publisher","first-page":"208","DOI":"10.1186\/1471-2105-14-208","volume":"14","author":"AJ Jimeno-Yepes","year":"2013","unstructured":"Jimeno-Yepes, A. J., Plaza, L., Mork, J. G., Aronson, A. R., & D\u00edaz, A. (2013). MeSH indexing based on automatically generated summaries. BMC Bioinformatics, 14, 208.","journal-title":"BMC Bioinformatics"},{"key":"9399_CR56","unstructured":"Jing, H., Barzilay, R., McKeown, K. & Elhadad, M. (1998). Summarization evaluation methods: Experiments and analysis. In AAAI symposium on intelligent summarization (pp. 51\u20139)."},{"key":"9399_CR57","doi-asserted-by":"publisher","unstructured":"Kabadjov, M., Steinberger, J., Barker, E., Kruschwitz, U., & Poesio, M. (2015). Onforums: The shared task on online forum summarisation at multiling\u201915. In Proceedings of the 7th forum for information retrieval evaluation, FIRE \u201915. ACM, New York, NY, USA (pp. 21\u201326). doi: 10.1145\/2838706.2838709 .","DOI":"10.1145\/2838706.2838709"},{"key":"9399_CR58","doi-asserted-by":"crossref","unstructured":"Katragadda, R. (2010). GEMS: Generative modeling for evaluation of summaries. In Proceedings of the 11th international conference on computational linguistics and intelligent text processing (pp. 724\u201335).","DOI":"10.1007\/978-3-642-12116-6_61"},{"key":"9399_CR59","doi-asserted-by":"publisher","first-page":"737","DOI":"10.1016\/j.asoc.2015.01.070","volume":"30","author":"A Khan","year":"2015","unstructured":"Khan, A., Salim, N., & Kumar, Y. J. (2015). A framework for multi-document abstractive summarization based on semantic role labelling. Applied Soft Computing, 30, 737\u2013747.","journal-title":"Applied Soft Computing"},{"key":"9399_CR60","doi-asserted-by":"crossref","unstructured":"Koehn, P., Hoang, H., Birch, A., Callison-Burch, C., Federico, M., Bertoldi, N., Cowan, B., Shen, W., Moran, C., Zens, R., & et al. (2007). Moses: Open source toolkit for statistical machine translation. In Proceedings of the 45th annual meeting of the acl on interactive poster and demonstration sessions. Association for Computational Linguistics (pp. 177\u201380).","DOI":"10.3115\/1557769.1557821"},{"key":"9399_CR61","doi-asserted-by":"crossref","unstructured":"Kupiec, J., Pedersen, J., & Chen, F., (1995). A trainable document summarizer. In Proceedings of the 18th annual international ACM SIGIR conference on research and development in information retrieval. ACM (pp. 68\u201373).","DOI":"10.1145\/215206.215333"},{"key":"9399_CR62","unstructured":"Labeke, N.\u00a0V., Whitelock, D., Field, D., Pulman, S., & Richardson, J. (2013a). What is my essay really saying? Using extractive summarization to motivate reflection and redrafting. In Proceedings of the workshops at the 16th international conference on artificial intelligence in education AIED 2013, Memphis, USA, July 9\u201313. Vol. 1009 of CEUR workshop proceedings. CEUR-WS.org."},{"key":"9399_CR63","unstructured":"Labeke, N.\u00a0V., Whitelock, D., Field, D., Pulman, S., & Richardson, J. T.\u00a0E. (2013b). OpenEssayist: extractive summarisation and formative assessment of free-text essays. In 1st international workshop on discourse-centric learning analytics. A pre conference workshop at LAK13. http:\/\/oro.open.ac.uk\/37548\/ ."},{"key":"9399_CR64","unstructured":"Lapata, M., & Barzilay, R., (2005a). Automatic evaluation of text coherence: Models and representations. In Proceedings of the 19th international joint conference on artificial intelligence, Edinburgh (pp. 1085\u20131090)."},{"key":"9399_CR65","unstructured":"Lapata, M., & Barzilay, R. (2005b). Automatic evaluation of text coherence: Models and representations. In Proceedings of the 19th international joint conference on artificial intelligence. IJCAI\u201905. Morgan Kaufmann Publishers Inc., San Francisco, CA, USA (pp. 1085\u20131090). http:\/\/dl.acm.org\/citation.cfm?id=1642293.1642467 ."},{"key":"9399_CR66","unstructured":"Lin, C.-Y. (2001). Summary evaluation environment. http:\/\/www.isi.edu\/~cyl\/SEE ."},{"key":"9399_CR67","unstructured":"Lin, C.-Y. (2004a). Rouge: A package for automatic evaluation of summaries. In Text summarization branches out: Proceedings of the ACL-04 workshop. Association for Computational Linguistics, Barcelona, Spain (pp. 74\u201381)."},{"key":"9399_CR68","unstructured":"Lin, C.-Y. (2004b). ROUGE: A package for automatic evaluation of summaries. In Proceedings of association of computational linguistics text summarization workshop (pp. 74\u201381)."},{"key":"9399_CR69","doi-asserted-by":"crossref","unstructured":"Lin, C.-Y., & Hovy, E. (2002). Manual and automatic evaluation of summaries. In Proceedings of the workshop on automatic summarization post conference workshop of ACL-02 (DUC 2002).","DOI":"10.3115\/1118162.1118168"},{"key":"9399_CR70","doi-asserted-by":"publisher","unstructured":"Lin, C.-Y., & Hovy, E. (2003). Automatic evaluation of summaries using n-gram co-occurrence statistics. In Proceedings of the 2003 conference of the North American chapter of the association for computational linguistics on human language technology\u2014Volume 1. NAACL \u201903. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 71\u201378). doi: 10.3115\/1073445.1073465","DOI":"10.3115\/1073445.1073465"},{"key":"9399_CR71","unstructured":"Lin, Z., Liu, C., Ng, H.\u00a0T., & Kan, M.-Y. (2012). Combining coherence models and machine translation evaluation metrics for summarization evaluation. In Proceedings of the 50th annual meeting of the association for computational linguistics: Long papers\u2014Volume 1. Association for Computational Linguistics (pp. 1006\u20131014)."},{"key":"9399_CR72","unstructured":"Liseth, A. (2004). En evaluering av NorSum en automatisk tekstsammenfatter for norsk. Hovedfagsoppgave. Technical report: Universitetet i Bergen, Seksjon for lingvistiske fag."},{"key":"9399_CR73","doi-asserted-by":"crossref","unstructured":"Liu, F., & Liu, Y. (2008a). Correlation between rouge and human evaluation of extractive meeting summaries. In Proceedings of the 46th annual meeting of the association for computational linguistics on human language technologies: Short papers. HLT-Short \u201908. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 201\u2013204). http:\/\/dl.acm.org\/citation.cfm?id=1557690.1557747","DOI":"10.3115\/1557690.1557747"},{"key":"9399_CR74","doi-asserted-by":"crossref","unstructured":"Liu, F., & Liu, Y. (2008b). Correlation between ROUGE and human evaluation of extractive meeting summaries. In Proceedings of the 46th annual meeting of the association of computational linguistics: Human language technologies, short papers (pp. 201\u20134).","DOI":"10.3115\/1557690.1557747"},{"issue":"12","key":"9399_CR75","doi-asserted-by":"publisher","first-page":"1125","DOI":"10.1002\/int.20502","volume":"26","author":"E Lloret","year":"2011","unstructured":"Lloret, E., Llorens, H., Moreda, P., Saquete, E., & Palomar, M. (2011). Text summarization contribution to semantic question answering: New approaches for finding answers on the web. International Journal of Intelligent Systems, 26(12), 1125\u201352.","journal-title":"International Journal of Intelligent Systems"},{"issue":"1","key":"9399_CR76","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10462-011-9216-z","volume":"37","author":"E Lloret","year":"2012","unstructured":"Lloret, E., & Palomar, M. (2012). Text summarisation in progress: A literature review. Artificial Intelligence Review, 37(1), 1\u201341. doi: 10.1007\/s10462-011-9216-z .","journal-title":"Artificial Intelligence Review"},{"issue":"2","key":"9399_CR77","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1007\/s10579-012-9198-8","volume":"47","author":"E Lloret","year":"2013","unstructured":"Lloret, E., Plaza, L., & Aker, A. (2013). Analyzing the capabilities of crowdsourcing services for text summarization. Language Resources and Evaluation, 47(2), 337\u201369. doi: 10.1007\/s10579-012-9198-8 .","journal-title":"Language Resources and Evaluation"},{"key":"9399_CR78","unstructured":"Louis, A., & Nenkova, A. (2008). Automatic summary evaluation without human models. In Proceedings of the text analysing conference, (TAC 2008)."},{"key":"9399_CR79","doi-asserted-by":"crossref","unstructured":"Louis, A., & Nenkova, A. (2009a). Automatically evaluating content selection in summarization without human models. In Proceedings of the 2009 conference on empirical methods in natural language processing: Volume 1. Association for Computational Linguistics (pp. 306\u2013314).","DOI":"10.3115\/1699510.1699550"},{"key":"9399_CR80","unstructured":"Louis, A., & Nenkova, A. (2009b). Predicting summary quality using limited human input. In Proceedings of the 2009 text analysis conference (TAC 2009)."},{"key":"9399_CR81","doi-asserted-by":"publisher","DOI":"10.1075\/nlp.3","volume-title":"Automatic summarization","author":"I Mani","year":"2001","unstructured":"Mani, I. (2001). Automatic summarization (Vol. 3). Amsterdam: John Benjamins Publishing Company."},{"key":"9399_CR82","doi-asserted-by":"crossref","unstructured":"Mani, I., House, D., Klein, G., Hirschman, L., Firmin, T., & Sundheim, B. (1999). The TIPSTER SUMMAC text summarization evaluation. In Proceedings of the ninth conference on European chapter of the association for computational linguistics. Association for Computational Linguistics (pp. 77\u201385).","DOI":"10.3115\/977035.977047"},{"key":"9399_CR83","unstructured":"Marcu, D. (1997). From discourse structures to text summaries. In Proceedings of the ACL. Vol. 97 (pp. 82\u201388)."},{"key":"9399_CR84","doi-asserted-by":"crossref","unstructured":"Martschat, S., & Markert, K. (2017). Improving rouge for timeline summarization. In Proceedings of the 15th conference of the european chapter of the association for computational linguistics: Volume 2, short papers. Association for Computational Linguistics (pp. 285\u2013290).","DOI":"10.18653\/v1\/E17-2046"},{"key":"9399_CR85","doi-asserted-by":"publisher","first-page":"100","DOI":"10.1145\/1809400.1809422","volume":"11","author":"W Mason","year":"2010","unstructured":"Mason, W., & Watts, D. J. (2010). Financial incentives and the \u201cperformance of crowds\u201d. ACM SigKDD Explorations Newsletter, 11, 100\u20138.","journal-title":"ACM SigKDD Explorations Newsletter"},{"key":"9399_CR86","unstructured":"McKeown, K., Barzilay, R., Evans, D., Hatzivassiloglou, V., Kan, M.\u00a0Y., Schiffman, B., & Teufel, S. (2001). Columbia multi-document summarisation: Approach and evaluation. In Proceedings of the DUC 2001."},{"key":"9399_CR87","unstructured":"McKeown, K., Passonneau, R., Elson, D., Nenkova, A., & Hirschberg, J. (2005). Do summaries help? A task-based evaluation of multi-document summarization. In 28th annual ACM SIGIR conference on research and development in information retrieval, ACM, Salvador, Brazil (pp. 210\u201317)."},{"key":"9399_CR88","doi-asserted-by":"crossref","unstructured":"Nenkova, A. (2006). Summarization evaluation for text and speech: Issues and approaches. In INTERSPEECH-2006, paper 2079-Wed1WeS.1.","DOI":"10.21437\/Interspeech.2006-429"},{"issue":"2\u20133","key":"9399_CR89","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1561\/1500000015","volume":"5","author":"A Nenkova","year":"2011","unstructured":"Nenkova, A., & McKeown, K. (2011). Automatic summarization. Foundations and Trends in Information Retrieval, 5(2\u20133), 103\u2013233. doi: 10.1561\/1500000015 .","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"9399_CR90","unstructured":"Nenkova, A., & Passonneau, R. (2004). Evaluating content selection in summarization: The pyramid method. In HLT-NAACL 2004: Main Proceedings (pp. 145\u201352). Association for Computational Linguistics, Boston, Massachusetts, USA."},{"issue":"2","key":"9399_CR91","first-page":"2","volume":"4","author":"A Nenkova","year":"2007","unstructured":"Nenkova, A., Passonneau, R., & McKeown, K. (2007). The pyramid method: Incorporating human content selection variation in summarization evaluation. ACM Transactions on Speech and Language Processing (TSLP), 4(2), 2\u201323.","journal-title":"ACM Transactions on Speech and Language Processing (TSLP)"},{"key":"9399_CR92","doi-asserted-by":"crossref","unstructured":"Ng, J.-P., & Abrecht, V. (2015a). Better summarization evaluation with word embeddings for rouge. In Proceedings of the 2015 conference on empirical methods in natural language processing. Association for Computational Linguistics, Lisbon, Portugal (pp. 1925\u20131930).","DOI":"10.18653\/v1\/D15-1222"},{"key":"9399_CR93","doi-asserted-by":"crossref","unstructured":"Ng, J.-P., & Abrecht, V. (2015b). Better summarization evaluation with word embeddings for rouge. In Proceedings of the 2015 conference on empirical methods in natural language processing. Association for Computational Linguistics, Lisbon, Portugal (pp. 1925\u20131930). http:\/\/aclweb.org\/anthology\/D15-1222 .","DOI":"10.18653\/v1\/D15-1222"},{"key":"9399_CR94","doi-asserted-by":"crossref","unstructured":"Ono, K., Sumita, K., & Miike, S. (1994). Abstract generation based on rhetorical structure extraction. In Proceedings of the 15th conference on Computational linguistics\u2014Volume 1. Association for Computational Linguistics (pp. 344\u201348).","DOI":"10.3115\/991886.991946"},{"key":"9399_CR95","unstructured":"Over, P., & Liggett, W. (2002). Introduction to DUC: An intrinsic evaluation of generic news text summarization systems. In Proceedings of DUC 2002."},{"key":"9399_CR96","doi-asserted-by":"crossref","unstructured":"Owczarzak, K. (2009). DEPEVAL(summ): Dependency-based evaluation for automatic summaries. In Proceedings of the joint conference of the 47th annual meeting of the ACL and the 4th international joint conference on natural language processing of the AFNLP (pp. 190\u201398).","DOI":"10.3115\/1687878.1687907"},{"key":"9399_CR97","unstructured":"Owczarzak, K., Conroy, J.\u00a0M., Dang, H.\u00a0T., & Nenkova, A. (2012a). An assessment of the accuracy of automatic evaluation in summarization. In Proceedings of workshop on evaluation metrics and system comparison for automatic summarization. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 1\u20139)."},{"key":"9399_CR98","unstructured":"Owczarzak, K., Conroy, J.\u00a0M., Dang, H.\u00a0T., & Nenkova, A. (2012b). An assessment of the accuracy of automatic evaluation in summarization. In Proceedings of workshop on evaluation metrics and system comparison for automatic summarization. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 1\u20139). http:\/\/dl.acm.org\/citation.cfm?id=2391258.2391259 ."},{"key":"9399_CR99","unstructured":"Owczarzak, K., & Dang, H.\u00a0T. (2011). Overview of the TAC 2011 summarization track: Guided task and AESOP task. In Proceedings of the text analysis conference (TAC)."},{"issue":"1","key":"9399_CR100","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1016\/0306-4573(90)90014-S","volume":"26","author":"CD Paice","year":"1990","unstructured":"Paice, C. D. (1990). Constructing literature abstracts by computer: Techniques and prospects. Information Processing & Management, 26(1), 171\u201386.","journal-title":"Information Processing & Management"},{"key":"9399_CR101","unstructured":"Papineni, K., Roukos, S., Ward, T., & Zhu, W.-J. (2002). BLEU: A method for automatic evaluation of machine translation. In Proceedings of 40th annual meeting of the association for computational linguistics (pp. 311\u2013318)."},{"key":"9399_CR102","unstructured":"Passonneau, R.\u00a0J., Chen, E., Guo, W., & Perin, D. (2013). Automated pyramid scoring of summaries using distributional semantics. In Proceedings of the 51st annual meeting of the association for computational linguistics (Volume 2: Short Papers). Association for Computational Linguistics, Sofia, Bulgaria (pp. 143\u2013147)."},{"key":"9399_CR103","unstructured":"Passonneau, R.\u00a0J., Nenkova, A., McKeown, K., & Sigelman, S. (2005). Applying the pyramid method in DUC 2005. In Proceedings of the document understanding conference (DUC 05), Vancouver, BC, Canada."},{"issue":"8","key":"9399_CR104","doi-asserted-by":"publisher","first-page":"2966","DOI":"10.1016\/j.eswa.2012.12.012","volume":"40","author":"JM Perea-Ortega","year":"2013","unstructured":"Perea-Ortega, J. M., Lloret, E., Ure\u00f1a L\u00f3pez, A., & Palomar, M. (2013). Application of text summarization techniques to the geographical information retrieval task. Expert Systems with Applications, 40(8), 2966\u201374. doi: 10.1016\/j.eswa.2012.12.012 .","journal-title":"Expert Systems with Applications"},{"key":"9399_CR105","unstructured":"Pitler, E., Louis, A., & Nenkova, A. (2010). Automatic evaluation of linguistic quality in multi-document summarization. In Proceedings of the 48th annual meeting of the association for computational linguistics. ACL \u201910. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 544\u2013554). http:\/\/dl.acm.org\/citation.cfm?id=1858681.1858737 ."},{"key":"9399_CR106","doi-asserted-by":"crossref","unstructured":"Pitler, E., & Nenkova, A. (2008). Revisiting readability: A unified framework for predicting text quality. In Proceedings of the 2008 conference on empirical methods in natural language processing (pp. 186\u201395).","DOI":"10.3115\/1613715.1613742"},{"key":"9399_CR107","unstructured":"Plaza, L. (2014). Comparing different knowledge sources for the automatic summarization of biomedical literature. Journal of Biomedical Informatics, 52, 319\u2013328, special Section: Methods in clinical research informatics. http:\/\/www.sciencedirect.com\/science\/article\/pii\/S1532046414001610 ."},{"key":"9399_CR108","unstructured":"Plaza, L., Stevenson, M., & D\u00edaz, A. (2010). Improving summarization of biomedical documents using word sense disambiguation. In Proceedings of the 2010 workshop on biomedical natural language processing. BioNLP \u201910. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 55\u201363). http:\/\/dl.acm.org\/citation.cfm?id=1869961.1869968 ."},{"key":"9399_CR109","unstructured":"Radev, D.\u00a0R. (2001). Experiments in single and multidocument summarization using mead. In First document understanding conference (DUC 2001)."},{"key":"9399_CR110","doi-asserted-by":"crossref","unstructured":"Radev, D.\u00a0R., & Tam, D. (2003). Summarization evaluation using relative utility. In CIKM \u201903: Proceedings of the 12th international conference on information and knowledge management (pp. 508\u201311).","DOI":"10.1145\/956863.956960"},{"key":"9399_CR111","unstructured":"Rankel, P., Conroy, J.\u00a0M., Slud, E.\u00a0V., & O\u2019Leary, D.\u00a0P. (2011). Ranking human and machine summarization systems. In Proceedings of the conference on empirical methods in natural language processing. EMNLP \u201911. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 467\u2013473)."},{"key":"9399_CR112","unstructured":"Rankel, P.\u00a0A., Conroy, J.\u00a0M., Dang, H.\u00a0T., & Nenkova, A. (2013). A decade of automatic content evaluation of news summaries: Reassessing the state of the art. In Proceedings of the 51st annual meeting of the association for computational linguistics, ACL 2013, 4\u20139 August 2013, Sofia, Bulgaria, Volume 2: Short Papers (pp. 131\u2013136)."},{"key":"9399_CR113","doi-asserted-by":"crossref","unstructured":"Rankel, P.\u00a0A., Conroy, J.\u00a0M., & Schlesinger, J.\u00a0D. (2012). Better metrics to automatically predict the quality of a text summary. Algorithms, 5(4), 398. http:\/\/www.mdpi.com\/1999-4893\/5\/4\/398 .","DOI":"10.3390\/a5040398"},{"key":"9399_CR114","unstructured":"Reeve, L.\u00a0H., Han, H., & Brooks, A.\u00a0D. (2007). The use of domain-specific concepts in biomedical text summarization. Information Processing & Management, 43(6), 1765\u20131776, text summarization. http:\/\/www.sciencedirect.com\/science\/article\/pii\/S030645730700074X ."},{"key":"9399_CR115","unstructured":"Resnik, P. (1995). Using information content to evaluate semantic similarity in a taxonomy. In Proceedings of international joint conferences on artificial intelligence (IJCAI), Montreal, Canada (pp. 448\u201353)."},{"key":"9399_CR116","unstructured":"Saggion, H., & Lapalme, G. (2000). Selective analysis for automatic abstracting: Evaluating indicativeness and acceptability. In Proceedings of content-based multimedia information access (pp. 747\u201364)."},{"key":"9399_CR117","unstructured":"Saggion, H., & Szasz, S. (2012). The CONCISUS corpus of event summaries. In N. Calzolari, K. Choukri, T. Declerck, M. U. Dogan, B. Maegaard, J. Mariani, J. Odijk & S. Piperidis (Eds.), LREC. European Language Resources Association (ELRA) (pp. 2031\u201337)."},{"key":"9399_CR118","doi-asserted-by":"crossref","unstructured":"Saggion, H., Teufel, S., Radev, D., & Lam, W. (2002). Meta-evaluation of summaries in a cross-lingual environment using content-based metrics. In Proceedings of the 19th international conference on Computational linguistics (pp. 1\u20137).","DOI":"10.3115\/1072228.1072301"},{"key":"9399_CR119","unstructured":"Saggion, H., Torres-Moreno, J., da Cunha, I., SanJuan, E., & Vel\u00e1zquez-Morales, P. (2010). Multilingual summarization evaluation without human models. In COLING 2010, 23rd international conference on computational linguistics, posters volume, 23\u201327 August 2010, Beijing, China (pp. 1059\u20131067)."},{"key":"9399_CR120","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1016\/S0306-4573(96)00062-3","volume":"33","author":"G Salton","year":"1997","unstructured":"Salton, G., Singhal, A., Mitra, M., & Buckley, C. (1997). Automatic text structuring and summarization. Information Processing & Management., 33, 193\u2013207.","journal-title":"Information Processing & Management."},{"key":"9399_CR121","doi-asserted-by":"publisher","unstructured":"Schlesinger, J.\u00a0D., O\u2019Leary, D.\u00a0P., & Conroy, J.\u00a0M. (2008). Arabic\/English multi-document summarization with CLASSY\u2014The past and the future, Springer, Berlin (pp. 568\u2013581). 10.1007\/978-3-540-78135-6_49 .","DOI":"10.1007\/978-3-540-78135-6_49"},{"key":"9399_CR122","doi-asserted-by":"crossref","unstructured":"Schluter, N. (2017). The limits of automatic summarisation according to rouge. In Proceedings of the 15th conference of the european chapter of the association for computational linguistics: Volume 2, short papers. Association for Computational Linguistics (pp. 41\u201345).","DOI":"10.18653\/v1\/E17-2007"},{"issue":"6","key":"9399_CR123","doi-asserted-by":"publisher","first-page":"1500","DOI":"10.1016\/j.ipm.2007.01.014","volume":"43","author":"J Sj\u00f6bergh","year":"2007","unstructured":"Sj\u00f6bergh, J. (2007). Older versions of the ROUGEeval summarization evaluation system were easier to fool. Information Processing & Management, 43(6), 1500\u20135.","journal-title":"Information Processing & Management"},{"key":"9399_CR124","unstructured":"Smith, C., Danielsson, H., & J\u00f6nsson, A. (2012). A more cohesive summarizer. In COLING 2012, 24th international conference on computational linguistics, proceedings of the conference: Posters, 8\u201315 December 2012, Mumbai, India (pp. 1161\u20131170)."},{"issue":"6","key":"9399_CR125","doi-asserted-by":"publisher","first-page":"1449","DOI":"10.1016\/j.ipm.2007.03.009","volume":"43","author":"K Sp\u00e4rck Jones","year":"2007","unstructured":"Sp\u00e4rck Jones, K. (2007). Automatic summarising: The state of the art. Information Processing & Management, 43(6), 1449\u20131481. doi: 10.1016\/j.ipm.2007.03.009 .","journal-title":"Information Processing & Management"},{"key":"9399_CR126","unstructured":"Sparck Jones, K., & Galliers, J. (1996). Evaluating natural language processing systems (an analysis and review). In Lecture Notes in Computer Science, Springer."},{"key":"9399_CR127","unstructured":"Steinberger, J., Kabadjov, M., Pouliquen, B., Steinberger, R., & Poesio, M. (2009). WB-JRC-UT\u2019s participation in TAC 2009: Update summarization and AESOP tasks. In Proceedings of the 2009 text analysis conference (TAC 2009)."},{"key":"9399_CR128","unstructured":"Stewart, J. (2008). Genre oriented summarization. Ph.D. thesis. https:\/\/www.lti.cs.cmu.edu\/sites\/default\/files\/research\/thesis\/2008\/jade_goldstein_stewart_genre_oriented_summarization.pdf"},{"key":"9399_CR129","doi-asserted-by":"crossref","unstructured":"Tang, J., & Sanderson, M. (2010). Evaluation and user preference study on spatial diversity. In Proceedings of the 32nd European conference on information retrieval (ECIR).","DOI":"10.1007\/978-3-642-12275-0_18"},{"key":"9399_CR130","unstructured":"Teufel, S. (2001). Task-based evaluation of summary quality: Describing relationships between scientific papers. In Workshop automatic summarization, NAACL (pp. 12\u201321)."},{"key":"9399_CR131","unstructured":"Teufel, S., & van Halteren, H. (2004). Evaluating information content by factoid analysis: Human annotation and stability. In Proceedings of the conference on empirical methods in natural language processing (pp. 419\u201326)."},{"key":"9399_CR132","doi-asserted-by":"crossref","unstructured":"Tombros, A., & Sanderson, M. (1998). Advantages of query biased summaries in information retrieval. In Proceedings of the 21st annual international ACM SIGIR conference on research and development in information retrieval, ACM, New York, NY, USA (pp. 2\u201310).","DOI":"10.1145\/290941.290947"},{"key":"9399_CR133","unstructured":"Torres-Moreno, J. (2011). R\u00e9sum\u00e9 automatique de documents. Recherche d\u2019information et web. Hermes Science Publications. https:\/\/books.google.es\/books?id=9HeLsuRFRJMC ."},{"key":"9399_CR134","doi-asserted-by":"crossref","unstructured":"Torres-Moreno, J. (2014). Automatic Text Summarization. Cognitive science and knowledge management series. Wiley. https:\/\/books.google.es\/books?id=aPHsBQAAQBAJ .","DOI":"10.1002\/9781119004752"},{"key":"9399_CR135","doi-asserted-by":"publisher","first-page":"13","DOI":"10.17562\/PB-42-2","volume":"42","author":"J Torres-Moreno","year":"2010","unstructured":"Torres-Moreno, J., Saggion, H., da Cunha, I., SanJuan, E., & Vel\u00e1zquez-Morales, P. (2010a). Summary evaluation with and without references. Polibits: Research Journal on Computer Science and Computer Engineering with Applications, 42, 13\u201319.","journal-title":"Polibits: Research Journal on Computer Science and Computer Engineering with Applications"},{"key":"9399_CR136","unstructured":"Torres-Moreno, J., Saggion, H., da\u00a0Cunha, I., Vel\u00e1zquez-Morales, P., & SanJuan, E. (2010b). Evaluation automatique de r\u00e9sum\u00e9s avec et sans r\u00e9f\u00e9rences. In TALN\u201910, Montr\u00e9al, Canada."},{"key":"9399_CR137","unstructured":"Tratz, S., & Hovy, E. (2008). Summarization evaluation using transformed basic elements. In Proceedings of the 1st text analysis conference."},{"key":"9399_CR138","doi-asserted-by":"crossref","unstructured":"Turchi, M., Steinberger, J., Kabadjov, M., & Steinberger, R. (2010). Using parallel corpora for multilingual (multi-document) summarisation evaluation. In Multilingual and multimodal information access evaluation. Vol. 6360 of Lecture Notes in Computer Science (pp. 52\u201363).","DOI":"10.1007\/978-3-642-15998-5_7"},{"key":"9399_CR139","unstructured":"Ulrich, J., Murray, G., & Carenini, G. (2008). A publicly available annotated corpus for supervised email summarization. In AAAI08 EMAIL Workshop, AAAI, Chicago, USA."},{"key":"9399_CR140","doi-asserted-by":"crossref","unstructured":"Vadlapudi, R., & Katragadda, R. (2010a). Quantitative evaluation of grammaticality of summaries. In Proceedings of the 11th international conference on computational linguistics and intelligent text processing, CICLing 2010, Iasi, Romania (pp. 736\u201347).","DOI":"10.1007\/978-3-642-12116-6_62"},{"key":"9399_CR141","unstructured":"Vadlapudi, R., & Katragadda, R. (2010b). On automated evaluation of readability of summaries: Capturing grammaticality, focus, structure and coherence. In Proceedings of the NAACL HLT 2010 student research workshop. HLT-SRWS \u201910. Association for Computational Linguistics, Stroudsburg, PA, USA (pp. 7\u201312). http:\/\/dl.acm.org\/citation.cfm?id=1858146.1858148 ."},{"key":"9399_CR142","unstructured":"Vadlapudi, R., & Katragadda, R. (2010c). On automated evaluation of readability of summaries: Capturing grammaticality, focus, structure and coherence. In Proceedings of the NAACL HLT 2010 student research workshop (pp. 7\u201312)."},{"key":"9399_CR143","volume-title":"Some aspects of text grammars. A study in theoretical linguistics and poetics","author":"T Dijk Van","year":"1972","unstructured":"Van Dijk, T. (1972). Some aspects of text grammars. A study in theoretical linguistics and poetics. Paris, Mouton: The Hague."},{"key":"9399_CR144","unstructured":"Voorhees, E. (2003). Overview of the TREC 2003 question answering track. In Proceedings of the twelfth text retrieval conference (TREC)."},{"key":"9399_CR145","doi-asserted-by":"crossref","unstructured":"Wang, C., Long, L., & Li, L. (2008). HowNet based evaluation for chinese text summarization. In Proceedings of the international conference on natural language processing and software engineering (pp. 82\u20137).","DOI":"10.1109\/NLPKE.2008.4906789"},{"key":"9399_CR146","unstructured":"Wang, X., Evanini, K., & Zechner, K. (2013). Coherence modeling for the automated assessment of spontaneous spoken responses. In Proceedings of the 2013 conference of the North American chapter of the association for computational linguistics: Human language technologies. Association for Computational Linguistics, Atlanta, Georgia (pp. 814\u2013819). http:\/\/www.aclweb.org\/anthology\/N13-1101 ."},{"key":"9399_CR147","unstructured":"Wu, M., Wilkinson, R., & Paris, C. (2004). An evaluation on query-biased summarisation for the question answering task. In Proceedings of the Australasian language technology workshop 2004, Sydney, Australia (pp. 32\u20138). http:\/\/www.aclweb.org\/anthology\/U\/U04\/U04-1005 ."},{"key":"9399_CR148","doi-asserted-by":"crossref","unstructured":"Yin, W., & Sch\u00fctze, H. (2015). Discriminative phrase embedding for paraphrase identification. In Proceedings of the 2015 conference of the North American chapter of the association for computational linguistics: Human language technologies. Association for Computational Linguistics, Denver, Colorado (pp. 1368\u20131373).","DOI":"10.3115\/v1\/N15-1154"},{"key":"9399_CR149","doi-asserted-by":"crossref","unstructured":"Zhou, L., Lin, C.-Y., Munteanu, D.\u00a0S., & Hovy, E. (2006). ParaEval: Using paraphrases to evaluate summaries automatically. In Proceedings of the human language technology\/North American association of computational linguistics conference (pp. 447\u201354).","DOI":"10.3115\/1220835.1220892"},{"key":"9399_CR150","doi-asserted-by":"publisher","first-page":"221","DOI":"10.1016\/j.compbiomed.2013.12.006","volume":"59","author":"X Zhu","year":"2013","unstructured":"Zhu, X., & Cimino, J. J. (2013). Clinicians\u2019 evaluation of computer-assisted medication summarization of electronic medical records. Computers in Biology and Medicine, 59, 221\u2013231.","journal-title":"Computers in Biology and Medicine"}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-017-9399-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-017-9399-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-017-9399-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,26]],"date-time":"2024-06-26T17:38:07Z","timestamp":1719423487000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-017-9399-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,9,2]]},"references-count":150,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2018,3]]}},"alternative-id":["9399"],"URL":"https:\/\/doi.org\/10.1007\/s10579-017-9399-2","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,9,2]]},"assertion":[{"value":"2 September 2017","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}