{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,23]],"date-time":"2025-10-23T21:06:48Z","timestamp":1761253608383,"version":"3.40.3"},"publisher-location":"Cham","reference-count":46,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030997359"},{"type":"electronic","value":"9783030997366"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99736-6_21","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T19:02:47Z","timestamp":1649098967000},"page":"304-321","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["WIDAR - Weighted Input Document Augmented ROUGE"],"prefix":"10.1007","author":[{"given":"Raghav","family":"Jain","sequence":"first","affiliation":[]},{"given":"Vaibhav","family":"Mavi","sequence":"additional","affiliation":[]},{"given":"Anubhav","family":"Jangra","sequence":"additional","affiliation":[]},{"given":"Sriparna","family":"Saha","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"21_CR1","doi-asserted-by":"crossref","unstructured":"Bhandari, M., Narayan Gour, P., Ashfaq, A., Liu, P., Neubig, G.: Re-evaluating evaluation in text summarization. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.751"},{"key":"21_CR2","doi-asserted-by":"crossref","unstructured":"B\u00f6hm, F., Gao, Y., Meyer, C.M., Shapira, O., Dagan, I., Gurevych, I.: Better rewards yield better summaries: learning to summarise without references. arXiv:abs\/1909.01214 (2019)","DOI":"10.18653\/v1\/D19-1307"},{"key":"21_CR3","doi-asserted-by":"crossref","unstructured":"Chen, P., Wu, F., Wang, T.: A semantic QA-based approach for text summarization evaluation. In: AAAI (2018)","DOI":"10.1609\/aaai.v32i1.11911"},{"key":"21_CR4","doi-asserted-by":"publisher","unstructured":"Chopra, S., Auli, M., Rush, A.M.: Abstractive sentence summarization with attentive recurrent neural networks. In: Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, San Diego, California, June 2016, pp. 93\u201398. Association for Computational Linguistics (2016). https:\/\/doi.org\/10.18653\/v1\/N16-1012. https:\/\/aclanthology.org\/N16-1012","DOI":"10.18653\/v1\/N16-1012"},{"key":"21_CR5","doi-asserted-by":"publisher","unstructured":"Clark, E., Celikyilmaz, A., Smith, N.A.: Sentence mover\u2019s similarity: automatic evaluation for multi-sentence texts. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, Florence, Italy, July 2019. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/P19-1264. https:\/\/aclanthology.org\/P19-1264","DOI":"10.18653\/v1\/P19-1264"},{"key":"21_CR6","unstructured":"Dang, H.T.: Overview of DUC 2005. In: Proceedings of the Document Understanding Conference, vol. 2005, pp. 1\u201312 (2005)"},{"key":"21_CR7","doi-asserted-by":"crossref","unstructured":"Duan, Y., Jatowt, A.: Across-time comparative summarization of news articles. In: Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining, pp. 735\u2013743. ACM (2019)","DOI":"10.1145\/3289600.3291008"},{"key":"21_CR8","doi-asserted-by":"publisher","first-page":"264","DOI":"10.1145\/321510.321519","volume":"16","author":"HP Edmundson","year":"1969","unstructured":"Edmundson, H.P.: New methods in automatic extracting. J. ACM 16, 264\u2013285 (1969)","journal-title":"J. ACM"},{"key":"21_CR9","doi-asserted-by":"publisher","unstructured":"Eyal, M., Baumel, T., Elhadad, M.: Question answering as an automatic evaluation metric for news article summarization. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), Minneapolis, Minnesota, June 2019, pp. 3938\u20133948. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/N19-1395. https:\/\/aclanthology.org\/N19-1395","DOI":"10.18653\/v1\/N19-1395"},{"key":"21_CR10","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1162\/tacl_a_00373","volume":"9","author":"AR Fabbri","year":"2021","unstructured":"Fabbri, A.R., Kryscinski, W., McCann, B., Socher, R., Radev, D.: SummEval: re-evaluating summarization evaluation. Trans. Assoc. Comput. Linguist. 9, 391\u2013409 (2021)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"21_CR11","unstructured":"Ganesan, K.A.: Rouge 2.0: updated and improved measures for evaluation of summarization tasks. arXiv:abs\/1803.01937 (2018)"},{"key":"21_CR12","doi-asserted-by":"publisher","unstructured":"Gao, Y., Zhao, W., Eger, S.: SUPERT: towards new frontiers in unsupervised evaluation metrics for multi-document summarization. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. Association for Computational Linguistics, July 2020. https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.124. https:\/\/aclanthology.org\/2020.acl-main.124","DOI":"10.18653\/v1\/2020.acl-main.124"},{"key":"21_CR13","unstructured":"Giannakopoulos, G., Karkaletsis, V.: AutoSumMENG and MeMoG in evaluating guided summaries. Theory Appl. Categ. (2011)"},{"key":"21_CR14","doi-asserted-by":"crossref","unstructured":"Grusky, M., Naaman, M., Artzi, Y.: Newsroom: a dataset of 1.3 million summaries with diverse extractive strategies. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers), New Orleans, Louisiana, June 2018. Association for Computational Linguistics (2018). https:\/\/aclanthology.org\/N18-1065","DOI":"10.18653\/v1\/N18-1065"},{"key":"21_CR15","doi-asserted-by":"crossref","unstructured":"Hasan, T., et al.: XL-Sum: large-scale multilingual abstractive summarization for 44 languages. In: Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021, pp. 4693\u20134703 (2021)","DOI":"10.18653\/v1\/2021.findings-acl.413"},{"key":"21_CR16","unstructured":"Jangra, A., Jain, R., Mavi, V., Saha, S., Bhattacharyya, P.: Semantic extractor-paraphraser based abstractive summarization. arXiv preprint arXiv:2105.01296 (2021)"},{"key":"21_CR17","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"190","DOI":"10.1007\/978-3-030-45442-5_24","volume-title":"Advances in Information Retrieval","author":"A Jangra","year":"2020","unstructured":"Jangra, A., Jatowt, A., Hasanuzzaman, M., Saha, S.: Text-image-video summary generation using joint integer linear programming. In: Jose, J.M., et al. (eds.) ECIR 2020. LNCS, vol. 12036, pp. 190\u2013198. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45442-5_24"},{"key":"21_CR18","unstructured":"Jangra, A., Jatowt, A., Saha, S., Hasanuzzaman, M.: A survey on multi-modal summarization (2021)"},{"key":"21_CR19","doi-asserted-by":"crossref","unstructured":"Jangra, A., Saha, S., Jatowt, A., Hasanuzzaman, M.: Multi-modal summary generation using multi-objective optimization. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1745\u20131748 (2020)","DOI":"10.1145\/3397271.3401232"},{"key":"21_CR20","doi-asserted-by":"crossref","unstructured":"Jangra, A., Saha, S., Jatowt, A., Hasanuzzaman, M.: Multi-modal supplementary-complementary summarization using multi-objective optimization. In: Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 818\u2013828 (2021)","DOI":"10.1145\/3404835.3462877"},{"key":"21_CR21","doi-asserted-by":"publisher","unstructured":"Kryscinski, W., Keskar, N.S., McCann, B., Xiong, C., Socher, R.: Neural text summarization: a critical evaluation. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), Hong Kong, China, November 2019. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1051. https:\/\/aclanthology.org\/D19-1051","DOI":"10.18653\/v1\/D19-1051"},{"key":"21_CR22","doi-asserted-by":"publisher","unstructured":"Kupiec, J., Pedersen, J., Chen, F.: A trainable document summarizer. In: Proceedings of the 18th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 1995, New York, NY, USA, pp. 68\u201373. Association for Computing Machinery (1995). https:\/\/doi.org\/10.1145\/215206.215333","DOI":"10.1145\/215206.215333"},{"key":"21_CR23","unstructured":"Kusner, M., Sun, Y., Kolkin, N., Weinberger, K.: From word embeddings to document distances. In: Bach, F., Blei, D. (eds.) Proceedings of the 32nd International Conference on Machine Learning. Proceedings of Machine Learning Research, Lille, France, 07\u201309 July 2015, vol. 37, pp. 957\u2013966. PMLR (2015). https:\/\/proceedings.mlr.press\/v37\/kusnerb15.html"},{"key":"21_CR24","doi-asserted-by":"crossref","unstructured":"Lavie, A., Agarwal, A.: METEOR: an automatic metric for MT evaluation with high levels of correlation with human judgments. In: Proceedings of the Second Workshop on Statistical Machine Translation, Prague, Czech Republic, June 2007, pp. 228\u2013231. Association for Computational Linguistics (2007). https:\/\/aclanthology.org\/W07-0734","DOI":"10.3115\/1626355.1626389"},{"key":"21_CR25","unstructured":"Lin, C.Y.: ROUGE: a package for automatic evaluation of summaries. In: Text Summarization Branches Out, Barcelona, Spain, July 2004, pp. 74\u201381. Association for Computational Linguistics (2004). https:\/\/aclanthology.org\/W04-1013"},{"key":"21_CR26","doi-asserted-by":"crossref","unstructured":"Nallapati, R., Zhou, B., Santos, C.D., \u00c7aglar G\u00fcl\u00e7ehre, Xiang, B.: Abstractive text summarization using sequence-to-sequence RNNs and beyond. In: CoNLL (2016)","DOI":"10.18653\/v1\/K16-1028"},{"key":"21_CR27","doi-asserted-by":"crossref","unstructured":"Nenkova, A.: Summarization evaluation for text and speech: issues and approaches. In: INTERSPEECH (2006)","DOI":"10.21437\/Interspeech.2006-429"},{"key":"21_CR28","doi-asserted-by":"publisher","unstructured":"Ng, J.P., Abrecht, V.: Better summarization evaluation with word embeddings for ROUGE. In: Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, Lisbon, Portugal, September 2015, pp. 1925\u20131930. Association for Computational Linguistics (2015). https:\/\/doi.org\/10.18653\/v1\/D15-1222. https:\/\/aclanthology.org\/D15-1222","DOI":"10.18653\/v1\/D15-1222"},{"key":"21_CR29","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1016\/0306-4573(90)90014-S","volume":"26","author":"CD Paice","year":"1990","unstructured":"Paice, C.D.: Constructing literature abstracts by computer: techniques and prospects. Inf. Process. Manag. 26, 171\u2013186 (1990)","journal-title":"Inf. Process. Manag."},{"key":"21_CR30","doi-asserted-by":"crossref","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: BLEU: a method for automatic evaluation of machine translation. In: ACL (2002)","DOI":"10.3115\/1073083.1073135"},{"key":"21_CR31","unstructured":"Passonneau, R.J., Chen, E., Guo, W., Perin, D.: Automated pyramid scoring of summaries using distributional semantics. In: Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers), Sofia, Bulgaria, August 2013, pp. 143\u2013147. Association for Computational Linguistics (2013). https:\/\/aclanthology.org\/P13-2026"},{"key":"21_CR32","doi-asserted-by":"publisher","unstructured":"Peyrard, M., Botschen, T., Gurevych, I.: Learning to score system summaries for better content selection evaluation. In: Proceedings of the Workshop on New Frontiers in Summarization, Copenhagen, Denmark, September 2017. Association for Computational Linguistics (2017). https:\/\/doi.org\/10.18653\/v1\/W17-4510. https:\/\/aclanthology.org\/W17-4510","DOI":"10.18653\/v1\/W17-4510"},{"key":"21_CR33","doi-asserted-by":"publisher","unstructured":"Popovi\u0107, M.: chrF: character n-gram F-score for automatic MT evaluation. In: Proceedings of the Tenth Workshop on Statistical Machine Translation, Lisbon, Portugal, September 2015, pp. 392\u2013395. Association for Computational Linguistics (2015). https:\/\/doi.org\/10.18653\/v1\/W15-3049. https:\/\/aclanthology.org\/W15-3049","DOI":"10.18653\/v1\/W15-3049"},{"key":"21_CR34","unstructured":"Rath, G.J., Resnick, S., Savage, T.R.: The formation of abstracts by the selection of sentences (1961)"},{"key":"21_CR35","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1016\/j.knosys.2018.10.021","volume":"164","author":"N Saini","year":"2019","unstructured":"Saini, N., Saha, S., Jangra, A., Bhattacharyya, P.: Extractive single document summarization using multi-objective optimization: exploring self-organized differential evolution, grey wolf optimizer and water cycle algorithm. Knowl. Based Syst. 164, 45\u201367 (2019)","journal-title":"Knowl. Based Syst."},{"key":"21_CR36","doi-asserted-by":"crossref","unstructured":"Scialom, T., Dray, P.A., Lamprier, S., Piwowarski, B., Staiano, J.: MLSUM: the multilingual summarization corpus. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 8051\u20138067 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.647"},{"key":"21_CR37","doi-asserted-by":"publisher","unstructured":"Scialom, T., Lamprier, S., Piwowarski, B., Staiano, J.: Answers unite! Unsupervised metrics for reinforced summarization models. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), Hong Kong, China, November 2019. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1320. https:\/\/aclanthology.org\/D19-1320","DOI":"10.18653\/v1\/D19-1320"},{"key":"21_CR38","doi-asserted-by":"crossref","unstructured":"See, A., Liu, P., Manning, C.: Get to the point: summarization with pointer-generator networks. In: Association for Computational Linguistics (2017). https:\/\/arxiv.org\/abs\/1704.04368","DOI":"10.18653\/v1\/P17-1099"},{"key":"21_CR39","doi-asserted-by":"crossref","unstructured":"ShafieiBavani, E., Ebrahimi, M., Wong, R., Chen, F.: A graph-theoretic summary evaluation for ROUGE. In: Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, Brussels, Belgium, October\u2013November 2018. Association for Computational Linguistics (2018). https:\/\/aclanthology.org\/D18-1085","DOI":"10.18653\/v1\/D18-1085"},{"key":"21_CR40","unstructured":"ShafieiBavani, E., Ebrahimi, M., Wong, R.K., Chen, F.: Summarization evaluation in the absence of human model summaries using the compositionality of word embeddings. In: COLING (2018)"},{"key":"21_CR41","doi-asserted-by":"publisher","unstructured":"Sun, S., Nenkova, A.: The feasibility of embedding based automatic evaluation for single document summarization. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), Hong Kong, China, November 2019, pp. 1216\u20131221. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1116. https:\/\/aclanthology.org\/D19-1116","DOI":"10.18653\/v1\/D19-1116"},{"key":"21_CR42","doi-asserted-by":"publisher","unstructured":"Vasilyev, O., Bohannon, J.: Is human scoring the best criteria for summary evaluation? In: Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021. Association for Computational Linguistics, August 2021. https:\/\/doi.org\/10.18653\/v1\/2021.findings-acl.192. https:\/\/aclanthology.org\/2021.findings-acl.192","DOI":"10.18653\/v1\/2021.findings-acl.192"},{"key":"21_CR43","doi-asserted-by":"crossref","unstructured":"Vedantam, R., Zitnick, C.L., Parikh, D.: CIDER: consensus-based image description evaluation. In: 2015 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 4566\u20134575 (2015)","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"21_CR44","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K.Q., Artzi, Y.: BERTScore: Evaluating text generation with BERT. In: International Conference on Learning Representations (2020). https:\/\/openreview.net\/forum?id=SkeHuCVFDr"},{"key":"21_CR45","doi-asserted-by":"publisher","unstructured":"Zhao, W., Peyrard, M., Liu, F., Gao, Y., Meyer, C.M., Eger, S.: MoverScore: text generation evaluating with contextualized embeddings and earth mover distance. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), Hong Kong, China, November 2019, pp. 563\u2013578. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1053. https:\/\/aclanthology.org\/D19-1053","DOI":"10.18653\/v1\/D19-1053"},{"key":"21_CR46","doi-asserted-by":"crossref","unstructured":"Zhou, L., Lin, C.Y., Munteanu, D.S., Hovy, E.: ParaEval: using paraphrases to evaluate summaries automatically. In: Proceedings of the Human Language Technology Conference of the NAACL, Main Conference, New York City, USA, June 2006, pp. 447\u2013454. Association for Computational Linguistics (2006). https:\/\/aclanthology.org\/N06-1057","DOI":"10.3115\/1220835.1220892"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99736-6_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T08:57:23Z","timestamp":1710233843000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99736-6_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997359","9783030997366"],"references-count":46,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99736-6_21","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}