{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T02:10:44Z","timestamp":1774318244374,"version":"3.50.1"},"publisher-location":"Cham","reference-count":48,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032212887","type":"print"},{"value":"9783032212894","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-21289-4_21","type":"book-chapter","created":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T01:03:13Z","timestamp":1774314193000},"page":"320-335","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Towards Quantitative Summarization Evaluation: An Integrated Atomic-Based Evaluation Framework and\u00a0Dataset for\u00a0Text Summarization"],"prefix":"10.1007","author":[{"given":"Yan","family":"Lei","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Suncong","family":"Zheng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Roberts","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liang","family":"Pang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lei","family":"He","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shuang","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yu","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Huawei","family":"Shen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xueqi","family":"Cheng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuanzhuo","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,3,25]]},"reference":[{"key":"21_CR1","unstructured":"Agarwal, S., et\u00a0al.: gpt-oss-120b & gpt-oss-20b model card. arXiv preprint arXiv:2508.10925 (2025)"},{"key":"21_CR2","doi-asserted-by":"crossref","unstructured":"Bhandari, M., Gour, P., Ashfaq, A., Liu, P., Neubig, G.: Re-evaluating evaluation in text summarization. arXiv preprint arXiv:2010.07100 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.751"},{"key":"21_CR3","doi-asserted-by":"crossref","unstructured":"Chen, D., Bolton, J., Manning, C.D.: A thorough examination of the CNN\/daily mail reading comprehension task. arXiv preprint arXiv:1606.02858 (2016)","DOI":"10.18653\/v1\/P16-1223"},{"key":"21_CR4","doi-asserted-by":"crossref","unstructured":"Chen, X., Chan, Z., Gao, S., Yu, M.H., Zhao, D., Yan, R.: Learning towards abstractive timeline summarization. In: IJCAI, pp. 4939\u20134945 (2019)","DOI":"10.24963\/ijcai.2019\/686"},{"key":"21_CR5","unstructured":"Chhun, C., Colombo, P., Clavel, C., Suchanek, F.M.: Of human criteria and automatic metrics: a benchmark of the evaluation of story generation. arXiv preprint arXiv:2208.11646 (2022)"},{"key":"21_CR6","unstructured":"Clement, C.B., Bierbaum, M., O\u2019Keeffe, K.P., Alemi, A.A.: On the use of arxiv as a dataset. arXiv preprint arXiv:1905.00075 (2019)"},{"issue":"3","key":"21_CR7","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1037\/met0000079","volume":"21","author":"JC De Winter","year":"2016","unstructured":"De Winter, J.C., Gosling, S.D., Potter, J.: Comparing the pearson and spearman correlation coefficients across distributions and sample sizes: a tutorial using simulations and empirical data. Psychol. Methods 21(3), 273 (2016)","journal-title":"Psychol. Methods"},{"key":"21_CR8","unstructured":"Duan, Y., Chen, Z., Wei, F., Zhou, M., Shum, H.Y.: Twitter topic summarization by ranking tweets using social influence and content quality. In: Proceedings of COLING 2012, pp. 763\u2013780 (2012)"},{"key":"21_CR9","unstructured":"Dubey, A., et\u00a0al.: The llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024)"},{"key":"21_CR10","unstructured":"Ernst, O., et al.: Proposition-level clustering for multi-document summarization. arXiv preprint arXiv:2112.08770 (2021)"},{"key":"21_CR11","doi-asserted-by":"crossref","unstructured":"Fabbri, A.R., Li, I., She, T., Li, S., Radev, D.R.: Multi-news: a large-scale multi-document summarization dataset and abstractive hierarchical model. arXiv preprint arXiv:1906.01749 (2019)","DOI":"10.18653\/v1\/P19-1102"},{"key":"21_CR12","unstructured":"Gao, M., Ruan, J., Sun, R., Yin, X., Yang, S., Wan, X.: Human-like summarization evaluation with chatgpt. arXiv preprint arXiv:2304.02554 (2023)"},{"key":"21_CR13","unstructured":"Goyal, T., Li, J.J., Durrett, G.: News summarization and evaluation in the era of GPT-3. arXiv preprint arXiv:2209.12356 (2022)"},{"key":"21_CR14","unstructured":"Guo, D., et\u00a0al.: Deepseek-r1: incentivizing reasoning capability in LLMs via reinforcement learning. arXiv preprint arXiv:2501.12948 (2025)"},{"key":"21_CR15","doi-asserted-by":"crossref","unstructured":"Hasan, T., et al.: Xl-sum: large-scale multilingual abstractive summarization for 44 languages. arXiv preprint arXiv:2106.13822 (2021)","DOI":"10.18653\/v1\/2021.findings-acl.413"},{"key":"21_CR16","unstructured":"Hermann, K.M., et al.: Teaching machines to read and comprehend. Adv. Neural Inf. Process. Syst. 28 (2015)"},{"key":"21_CR17","doi-asserted-by":"crossref","unstructured":"Kry\u015bci\u0144ski, W., Keskar, N.S., McCann, B., Xiong, C., Socher, R.: Neural text summarization: a critical evaluation. arXiv preprint arXiv:1908.08960 (2019)","DOI":"10.18653\/v1\/D19-1051"},{"key":"21_CR18","unstructured":"Lin, C.Y.: Rouge: a package for automatic evaluation of summaries. In: Text Summarization Branches Out, pp. 74\u201381 (2004)"},{"key":"21_CR19","doi-asserted-by":"crossref","unstructured":"Liu, Y., Iter, D., Xu, Y., Wang, S., Xu, R., Zhu, C.: G-eval: NLG evaluation using GPT-4 with better human alignment. arXiv preprint arXiv:2303.16634 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"21_CR20","doi-asserted-by":"crossref","unstructured":"Liu, Y., et al.: Benchmarking generation and evaluation capabilities of large language models for instruction controllable summarization. arXiv preprint arXiv:2311.09184 (2023)","DOI":"10.18653\/v1\/2024.findings-naacl.280"},{"key":"21_CR21","unstructured":"Liu, Y., et\u00a0al.: Revisiting the gold standard: grounding summarization evaluation with robust human evaluation. arXiv preprint arXiv:2212.07981 (2022)"},{"key":"21_CR22","unstructured":"Lu, D., et al.: Bbt-fin: comprehensive construction of Chinese financial domain pre-trained language model, corpus and benchmark. arXiv preprint arXiv:2302.09432 (2023)"},{"key":"21_CR23","doi-asserted-by":"crossref","unstructured":"Min, S., et al.: Factscore: fine-grained atomic evaluation of factual precision in long form text generation. arXiv preprint arXiv:2305.14251 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"21_CR24","doi-asserted-by":"crossref","unstructured":"Nallapati, R., Zhou, B., Gulcehre, C., Xiang, B., et\u00a0al.: Abstractive text summarization using sequence-to-sequence RNNS and beyond. arXiv preprint arXiv:1602.06023 (2016)","DOI":"10.18653\/v1\/K16-1028"},{"key":"21_CR25","doi-asserted-by":"crossref","unstructured":"Narayan, S., Cohen, S.B., Lapata, M.: Don\u2019t give me the details, just the summary! topic-aware convolutional neural networks for extreme summarization. arXiv preprint arXiv:1808.08745 (2018)","DOI":"10.18653\/v1\/D18-1206"},{"key":"21_CR26","unstructured":"OpenAI: Gpt-4 technical report (2023)"},{"key":"21_CR27","unstructured":"Pu, X., Gao, M., Wan, X.: Summarization is (almost) dead. arXiv preprint arXiv:2309.09558 (2023)"},{"issue":"324","key":"21_CR28","doi-asserted-by":"publisher","first-page":"1379","DOI":"10.1080\/01621459.1968.10480934","volume":"63","author":"PK Sen","year":"1968","unstructured":"Sen, P.K.: Estimates of the regression coefficient based on kendall\u2019s tau. J. Am. Stat. Assoc. 63(324), 1379\u20131389 (1968)","journal-title":"J. Am. Stat. Assoc."},{"key":"21_CR29","doi-asserted-by":"crossref","unstructured":"Shen, C., Cheng, L., Nguyen, X.P., You, Y., Bing, L.: Large language models are not yet human-level evaluators for abstractive summarization. arXiv preprint arXiv:2305.13091 (2023)","DOI":"10.18653\/v1\/2023.findings-emnlp.278"},{"key":"21_CR30","unstructured":"Shen, Y., Wan, X.: Opinsummeval: revisiting automated evaluation for opinion summarization. arXiv preprint arXiv:2310.18122 (2023)"},{"key":"21_CR31","unstructured":"Shukla, A., et al.: Legal case document summarization: extractive and abstractive methods and their evaluation. arXiv preprint arXiv:2210.07544 (2022)"},{"key":"21_CR32","doi-asserted-by":"crossref","unstructured":"Skopek, O., Aralikatte, R., Gooding, S., Carbune, V.: Towards better evaluation of instruction-following: a case-study in summarization. arXiv preprint arXiv:2310.08394 (2023)","DOI":"10.18653\/v1\/2023.conll-1.16"},{"key":"21_CR33","doi-asserted-by":"crossref","unstructured":"Song, H., Su, H., Shalyminov, I., Cai, J., Mansour, S.: Finesure: fine-grained summarization evaluation using LLMs. arXiv preprint arXiv:2407.00908 (2024)","DOI":"10.18653\/v1\/2024.acl-long.51"},{"key":"21_CR34","doi-asserted-by":"crossref","unstructured":"Str\u00f6hle, T., Campos, R., Jatowt, A.: Contrastive text summarization: a survey. Int. J. Data Sci. Anal., 1\u201315 (2023)","DOI":"10.1007\/s41060-023-00434-4"},{"issue":"4","key":"21_CR35","doi-asserted-by":"publisher","first-page":"353","DOI":"10.1007\/s41060-023-00434-4","volume":"18","author":"T Str\u00f6hle","year":"2024","unstructured":"Str\u00f6hle, T., Campos, R., Jatowt, A.: Contrastive text summarization: a survey. Int. J. Data Sci. Anal. 18(4), 353\u2013367 (2024)","journal-title":"Int. J. Data Sci. Anal."},{"key":"21_CR36","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s10742-011-0077-3","volume":"11","author":"S Sun","year":"2011","unstructured":"Sun, S.: Meta-analysis of cohen\u2019s kappa. Health Serv. Outcomes Res. Method. 11, 145\u2013163 (2011)","journal-title":"Health Serv. Outcomes Res. Method."},{"key":"21_CR37","doi-asserted-by":"crossref","unstructured":"Tam, D., Mascarenhas, A., Zhang, S., Kwan, S., Bansal, M., Raffel, C.: Evaluating the factual consistency of large language models through summarization. arXiv preprint arXiv:2211.08412 (2022)","DOI":"10.18653\/v1\/2023.findings-acl.322"},{"key":"21_CR38","unstructured":"Wang, Y., et al.: Self-instruct: aligning language models with self-generated instructions. arXiv preprint arXiv:2212.10560 (2022)"},{"key":"21_CR39","doi-asserted-by":"crossref","unstructured":"Wu, H., Zhan, M., Tan, H., Hou, Z., Liang, D., Song, L.: VCSUM: a versatile Chinese meeting summarization dataset. arXiv preprint arXiv:2305.05280 (2023)","DOI":"10.18653\/v1\/2023.findings-acl.377"},{"key":"21_CR40","unstructured":"Yang, A., et\u00a0al.: Qwen2 technical report. arXiv preprint arXiv:2407.10671 (2024)"},{"key":"21_CR41","unstructured":"Young, A., et\u00a0al.: Yi: open foundation models by 01. ai. arXiv preprint arXiv:2403.04652 (2024)"},{"key":"21_CR42","first-page":"27263","volume":"34","author":"W Yuan","year":"2021","unstructured":"Yuan, W., Neubig, G., Liu, P.: Bartscore: evaluating generated text as text generation. Adv. Neural. Inf. Process. Syst. 34, 27263\u201327277 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"21_CR43","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K.Q., Artzi, Y.: Bertscore: evaluating text generation with Bert. arXiv preprint arXiv:1904.09675 (2019)"},{"key":"21_CR44","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1162\/tacl_a_00632","volume":"12","author":"T Zhang","year":"2024","unstructured":"Zhang, T., Ladhak, F., Durmus, E., Liang, P., McKeown, K., Hashimoto, T.B.: Benchmarking large language models for news summarization. Trans. Assoc. Comput. Linguistics 12, 39\u201357 (2024)","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"21_CR45","unstructured":"Zhao, W., Ren, X., Hessel, J., Cardie, C., Choi, Y., Deng, Y.: Wildchat: 1m ChatGpt interaction logs in the wild. arXiv preprint arXiv:2405.01470 (2024)"},{"key":"21_CR46","unstructured":"Zheng, L., et\u00a0al.: LMSYS-chat-1m: a large-scale real-world LLM conversation dataset. arXiv preprint arXiv:2309.11998 (2023)"},{"key":"21_CR47","doi-asserted-by":"crossref","unstructured":"Zhong, M., et\u00a0al.: QMSUM: a new benchmark for query-based multi-domain meeting summarization. arXiv preprint arXiv:2104.05938 (2021)","DOI":"10.18653\/v1\/2021.naacl-main.472"},{"key":"21_CR48","unstructured":"Zmandar, N., El-Haj, M., Rayson, P., Litvak, M., Giannakopoulos, G., Pittaras, N., et\u00a0al.: The financial narrative summarisation shared task FNs 2021. In: Proceedings of the 3rd Financial Narrative Processing Workshop, pp. 120\u2013125 (2021)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-21289-4_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T01:03:19Z","timestamp":1774314199000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-21289-4_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032212887","9783032212894"],"references-count":48,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-21289-4_21","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"25 March 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Delft","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 March 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"48","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2026.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}