{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T18:32:38Z","timestamp":1768415558676,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,20]],"date-time":"2024-08-20T00:00:00Z","timestamp":1724112000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,20]]},"DOI":"10.1145\/3685650.3685671","type":"proceedings-article","created":{"date-parts":[[2024,9,18]],"date-time":"2024-09-18T10:19:28Z","timestamp":1726654768000},"page":"1-4","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Assessing the Reliability and Validity of the Measures for Automatic Text Summarization"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3497-5044","authenticated-orcid":false,"given":"Rafael Dueire","family":"Lins","sequence":"first","affiliation":[{"name":"Univ. Federal Rural de Pernambuco, Univesidade Federal de Pernambuco, Recife, Pernambuco, Brazil"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5035-9404","authenticated-orcid":false,"given":"Hil\u00e1rio","family":"Oliveira","sequence":"additional","affiliation":[{"name":"Instituto Federal do Esp\u00edrito Santo, Serra, Esp\u00edrito Santo Brazil"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6937-1956","authenticated-orcid":false,"given":"Steven J.","family":"Simske","sequence":"additional","affiliation":[{"name":"Colorado State University, Boulder, Colorado, U.S.A"}]}],"member":"320","published-online":{"date-parts":[[2024,9,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-023-00836-y"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/321510.321519"},{"key":"e_1_3_2_1_3_1","volume-title":"Automatic text summarization: A comprehensive survey. Expert systems with applications 165","author":"El-Kassas Wafaa S","year":"2021","unstructured":"Wafaa S El-Kassas, Cherif R Salama, Ahmed A Rafea, and Hoda K Mohamed. 2021. Automatic text summarization: A comprehensive survey. Expert systems with applications 165 (2021), 113679."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2018.12.011"},{"key":"e_1_3_2_1_5_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74--81."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019815"},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. of the ACM Symposium on Document Engineering 2020","author":"Lins Rafael Dueire","unstructured":"Rafael Dueire Lins, Rafael Ferreira de Mello, and Steve J. Simske. 2020. ACM DocEng'2020 Competition on Extractive Text Summarization. In Proc. of the ACM Symposium on Document Engineering 2020 (Virtual Event, CA, USA) (DocEng '20). Association for Computing Machinery, NY, USA, Article 3, 4 pages."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3342558.3345388"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1147\/rd.22.0159"},{"key":"e_1_3_2_1_10_1","volume-title":"Reliability vs. Validity in Research | Difference, Types and Examples. https:\/\/www.scribbr.com\/methodology\/reliability-vs-validity\/","author":"Middleton F","year":"2023","unstructured":"F Middleton. 2023. Reliability vs. Validity in Research | Difference, Types and Examples. https:\/\/www.scribbr.com\/methodology\/reliability-vs-validity\/ (2023)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3685650"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 2016 ACM Symposium on Document Engineering","author":"Oliveira Hil\u00e1rio","unstructured":"Hil\u00e1rio Oliveira, Rinaldo Lima, Rafael Dueire Lins, Fred Freitas, Marcelo Riss, and Steven J. Simske. 2016. Assessing Concept Weighting in Integer Linear Programming Based Single-document Summarization. In Proceedings of the 2016 ACM Symposium on Document Engineering (Vienna, Austria) (DocEng'16). ACM, New York, NY, USA, 205--208."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/BRACIS.2016.079"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICTAI.2017.00051"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 40th Annual Meeting on Association for Computational Linguistics","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. BLEU: A Method for Automatic Evaluation of Machine Translation. In Proceedings of the 40th Annual Meeting on Association for Computational Linguistics (Philadelphia, Pennsylvania) (ACL '02). Assoc. for Computational Linguistics, USA, 311--318."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1002\/j.1538-7305.1948.tb01338.x"},{"key":"e_1_3_2_1_17_1","volume-title":"Functional Applications of Text Analytics Systems","author":"Simske Steven","unstructured":"Steven Simske and Marie Vans. 2021. Functional Applications of Text Analytics Systems. River Publishers Series in Document Engineering."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1155\/2020\/9365340"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3052783"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning (ICML'20)","author":"Zhang Jingqing","unstructured":"Jingqing Zhang, Yao Zhao, Mohammad Saleh, and Peter J. Liu. 2020. PEGASUS: Pre-Training with Extracted Gap-Sentences for Abstractive Summarization. In Proceedings of the 37th International Conference on Machine Learning (ICML'20). JMLR.org, Article 1051, 12 pages."},{"key":"e_1_3_2_1_21_1","volume-title":"BERTScore: Evaluating Text Generation with BERT. In 8th International Conference on Learning Representations, ICLR 2020","author":"Zhang Tianyi","year":"2020","unstructured":"Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. 2020. BERTScore: Evaluating Text Generation with BERT. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020. OpenReview.net."},{"key":"e_1_3_2_1_22_1","volume-title":"Benchmarking large language models for news summarization. arXiv preprint arXiv:2301.13848","author":"Zhang Tianyi","year":"2023","unstructured":"Tianyi Zhang, Faisal Ladhak, Esin Durmus, Percy Liang, Kathleen McKeown, and Tatsunori B Hashimoto. 2023. Benchmarking large language models for news summarization. arXiv preprint arXiv:2301.13848 (2023)."}],"event":{"name":"DocEng '24: ACM Symposium on Document Engineering 2024","location":"San Jose CA USA","acronym":"DocEng '24","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Symposium on Document Engineering 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3685650.3685671","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3685650.3685671","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:17:26Z","timestamp":1750295846000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3685650.3685671"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,20]]},"references-count":22,"alternative-id":["10.1145\/3685650.3685671","10.1145\/3685650"],"URL":"https:\/\/doi.org\/10.1145\/3685650.3685671","relation":{},"subject":[],"published":{"date-parts":[[2024,8,20]]},"assertion":[{"value":"2024-09-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}