{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,28]],"date-time":"2026-03-28T03:41:33Z","timestamp":1774669293001,"version":"3.50.1"},"publisher-location":"Cham","reference-count":48,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030997359","type":"print"},{"value":"9783030997366","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99736-6_24","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"351-366","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":17,"title":["HC4: A New Suite of Test Collections for Ad Hoc CLIR"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1696-0407","authenticated-orcid":false,"given":"Douglas W.","family":"Oard","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"24_CR1","doi-asserted-by":"crossref","unstructured":"Abualsaud, M., Ghelani, N., Zhang, H., Smucker, M.D., Cormack, G.V., Grossman, M.R.: A system for efficient high-recall retrieval. In: The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval, pp. 1317\u20131320. ACM (2018)","DOI":"10.1145\/3209978.3210176"},{"key":"24_CR2","unstructured":"Bajaj, P., et al.: MS MARCO: a human generated machine reading comprehension dataset. arXiv preprint arXiv:1611.09268 (2016)"},{"key":"24_CR3","unstructured":"Baron, J., Losey, R., Berman, M.: Perspectives on predictive coding: and other advanced search methods for the legal practitioner. American Bar Association, Section of Litigation (2016). https:\/\/books.google.com\/books?id=TdJ2AQAACAAJ"},{"key":"24_CR4","doi-asserted-by":"crossref","unstructured":"Bonab, H., Sarwar, S.M., Allan, J.: Training effective neural CLIR by bridging the translation gap. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 9\u201318 (2020)","DOI":"10.1145\/3397271.3401035"},{"key":"24_CR5","doi-asserted-by":"publisher","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Sanderson, M., J\u00e4rvelin, K., Allan, J., Bruza, P. (eds.) SIGIR 2004: Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Sheffield, UK, 25\u201329 July 2004, pp. 25\u201332. ACM (2004). https:\/\/doi.org\/10.1145\/1008992.1009000","DOI":"10.1145\/1008992.1009000"},{"key":"24_CR6","doi-asserted-by":"crossref","unstructured":"Chi, Z., et al.: InfoXLM: an information-theoretic framework for cross-lingual language model pre-training. arXiv preprint arXiv:2007.07834 (2020)","DOI":"10.18653\/v1\/2021.naacl-main.280"},{"key":"24_CR7","unstructured":"Clough, P., Sanderson, M.: Evaluating the performance of information retrieval systems using test collections. Inf. Res. 18(2) (2013)"},{"key":"24_CR8","doi-asserted-by":"crossref","unstructured":"Conneau, A., et al.: Unsupervised cross-lingual representation learning at scale. arXiv preprint arXiv:1911.02116 (2019)","DOI":"10.18653\/v1\/2020.acl-main.747"},{"key":"24_CR9","doi-asserted-by":"crossref","unstructured":"Cormack, G.V., Grossman, M.R.: Evaluation of machine-learning protocols for technology-assisted review in electronic discovery. In: Proceedings of the 37th International ACM SIGIR Conference on Research & Development in Information Retrieval, pp. 153\u2013162 (2014)","DOI":"10.1145\/2600428.2609601"},{"key":"24_CR10","doi-asserted-by":"publisher","unstructured":"Cormack, G.V., et al.: Dynamic sampling meets pooling. In: Piwowarski, B., Chevalier, M., Gaussier, \u00c9., Maarek, Y., Nie, J., Scholer, F. (eds.) Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2019, Paris, France, 21\u201325 July, pp. 1217\u20131220 (2019). ACM (2019). https:\/\/doi.org\/10.1145\/3331184.3331354","DOI":"10.1145\/3331184.3331354"},{"key":"24_CR11","doi-asserted-by":"crossref","unstructured":"Costello, C., Yang, E., Lawrie, D., Mayfield, J.: Patapasco: a Python framework for cross-language information retrieval experiments. In: Proceedings of the 44th European Conference on Information Retrieval (ECIR) (2022)","DOI":"10.1007\/978-3-030-99739-7_33"},{"key":"24_CR12","unstructured":"Davis, M.W., Dunning, T.: A TREC evaluation of query translation methods for multi-lingual text retrieval. In: Harman, D.K. (ed.) Proceedings of The Fourth Text REtrieval Conference, TREC 1995, Gaithersburg, Maryland, USA, 1\u20133 November 1995. NIST Special Publication, vol. 500\u2013236. National Institute of Standards and Technology (NIST) (1995). http:\/\/trec.nist.gov\/pubs\/trec4\/papers\/nmsu.ps.gz"},{"key":"24_CR13","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)"},{"key":"24_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1007\/978-3-642-15754-7_2","volume-title":"Multilingual Information Access Evaluation I. Text Retrieval Experiments","author":"N Ferro","year":"2010","unstructured":"Ferro, N., Peters, C.: CLEF 2009 ad hoc track overview: TEL and persian tasks. In: Peters, C., Di Nunzio, G.M., Kurimo, M., Mandl, T., Mostefa, D., Pe\u00f1as, A., Roda, G. (eds.) CLEF 2009. LNCS, vol. 6241, pp. 13\u201335. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-15754-7_2"},{"key":"24_CR15","doi-asserted-by":"publisher","unstructured":"Ferro, N., Peters, C.: Information Retrieval Evaluation in a Changing World: Lessons Learned from 20 Years of CLEF, vol. 41. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-22948-1","DOI":"10.1007\/978-3-030-22948-1"},{"key":"24_CR16","unstructured":"Ghalandari, D.G., Hokamp, C., The Pham, N., Glover, J., Ifrim, G.: A large-scale multi-document summarization dataset from the Wikipedia current events portal. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics (ACL 2020), pp. 1302\u20131308 (2020)"},{"key":"24_CR17","unstructured":"Grossman, M.R., Cormack, G.V., Roegiest, A.: TREC 2016 total recall track overview. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of The Twenty-Fifth Text REtrieval Conference, TREC 2016, Gaithersburg, Maryland, USA, 15\u201318 November 2016. NIST Special Publication, vol. 500\u2013321. National Institute of Standards and Technology (NIST) (2016). http:\/\/trec.nist.gov\/pubs\/trec25\/papers\/Overview-TR.pdf"},{"key":"24_CR18","unstructured":"Hieber, F., Domhan, T., Denkowski, M., Vilar, D., Sokolov, A., Clifton, A., Post, M.: Sockeye: a toolkit for neural machine translation. arXiv preprint arXiv:1712.05690 (2017)"},{"key":"24_CR19","unstructured":"Hu, J., Ruder, S., Siddhant, A., Neubig, G., Firat, O., Johnson, M.: XTREME: a massively multilingual multi-task benchmark for evaluating cross-lingual generalisation. In: International Conference on Machine Learning, pp. 4411\u20134421. PMLR (2020)"},{"key":"24_CR20","doi-asserted-by":"publisher","unstructured":"J\u00e4rvelin, K., Kek\u00e4l\u00e4inen, J.: IR evaluation methods for retrieving highly relevant documents. In: Yannakoudakis, E.J., Belkin, N.J., Ingwersen, P., Leong, M. (eds.) SIGIR 2000: Proceedings of the 23rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, Athens, Greece, 24\u201328 July 2000, pp. 41\u201348. ACM (2000). https:\/\/doi.org\/10.1145\/345508.345545","DOI":"10.1145\/345508.345545"},{"key":"24_CR21","unstructured":"Kando, N., Kuriyama, K., Nozue, T., Eguchi, K., Kato, H., Hidaka, S.: Overview of IR tasks. In: Kando, N. (ed.) Proceedings of the First NTCIR Workshop on Research in Japanese Text Retrieval and Term Recognition, NTCIR-1, Tokyo, Japan, 30 August\u20131 September 1999. National Center for Science Information Systems (NACSIS) (1999). http:\/\/research.nii.ac.jp\/ntcir\/workshop\/OnlineProceedings\/IR-overview.pdf"},{"key":"24_CR22","unstructured":"Kanoulas, E., Li, D., Azzopardi, L., Spijker, R.: CLEF 2019 technology assisted reviews in empirical medicine overview. In: CEUR Workshop Proceedings, vol. 2380 (2019)"},{"key":"24_CR23","doi-asserted-by":"crossref","unstructured":"Landis, J.R., Koch, G.G.: The measurement of observer agreement for categorical data. Biometrics, 159\u2013174 (1977)","DOI":"10.2307\/2529310"},{"key":"24_CR24","doi-asserted-by":"publisher","unstructured":"Lewis, D.D.: A sequential algorithm for training text classifiers: corrigendum and additional data. In: SIGIR Forum, vol. 29, no. 2, pp. 13\u201319 (1995). https:\/\/doi.org\/10.1145\/219587.219592","DOI":"10.1145\/219587.219592"},{"issue":"4","key":"24_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-031-02181-7","volume":"14","author":"J Lin","year":"2021","unstructured":"Lin, J., Nogueira, R., Yates, A.: Pretrained transformers for text ranking: BERT and beyond. Synth. Lect. Hum. Lang. Technol. 14(4), 1\u2013325 (2021)","journal-title":"Synth. Lect. Hum. Lang. Technol."},{"key":"24_CR26","doi-asserted-by":"crossref","unstructured":"MacAvaney, S., Yates, A., Cohan, A., Goharian, N.: CEDR: contextualized embeddings for document ranking. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1101\u20131104 (2019)","DOI":"10.1145\/3331184.3331317"},{"key":"24_CR27","doi-asserted-by":"publisher","unstructured":"Majumder, P., et al.: The FIRE 2008 evaluation exercise. ACM Trans. Asian Lang. Inf. Process. 9(3), 10:1\u201310:24 (2010). https:\/\/doi.org\/10.1145\/1838745.1838747","DOI":"10.1145\/1838745.1838747"},{"key":"24_CR28","doi-asserted-by":"publisher","unstructured":"Moffat, A., Zobel, J.: Rank-biased precision for measurement of retrieval effectiveness. ACM Trans. Inf. Syst. 27(1), 2:1\u20132:27 (2008). https:\/\/doi.org\/10.1145\/1416950.1416952","DOI":"10.1145\/1416950.1416952"},{"key":"24_CR29","doi-asserted-by":"crossref","unstructured":"Nair, S., Galuscakova, P., Oard, D.W.: Combining contextualized and non-contextualized query translations to improve CLIR. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1581\u20131584 (2020)","DOI":"10.1145\/3397271.3401270"},{"key":"24_CR30","doi-asserted-by":"crossref","unstructured":"Nair, S., et al.: Transfer learning approaches for building cross-language dense retrieval models. In: Proceedings of the 44th European Conference on Information Retrieval (ECIR) (2022)","DOI":"10.1007\/978-3-030-99736-6_26"},{"issue":"2\u20133","key":"24_CR31","first-page":"99","volume":"7","author":"DW Oard","year":"2013","unstructured":"Oard, D.W., Webber, W.: Information retrieval for e-discovery. Inf. Retr. 7(2\u20133), 99\u2013237 (2013)","journal-title":"Inf. Retr."},{"key":"24_CR32","unstructured":"Roegiest, A., Cormack, G.V., Clarke, C.L.A., Grossman, M.R.: TREC 2015 total recall track overview. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of The Twenty-Fourth Text REtrieval Conference, TREC 2015, Gaithersburg, Maryland, USA, 17\u201320 November 2015. NIST Special Publication, vol. 500\u2013319. National Institute of Standards and Technology (NIST) (2015). https:\/\/trec.nist.gov\/pubs\/trec24\/papers\/Overview-TR.pdf"},{"issue":"5","key":"24_CR33","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Inf. Retr. 11(5), 447\u2013470 (2008). https:\/\/doi.org\/10.1007\/s10791-008-9059-7","journal-title":"Inf. Retr."},{"key":"24_CR34","doi-asserted-by":"publisher","unstructured":"Sakai, T., Oard, D.W., Kando, N.: Evaluating Information Retrieval and Access Tasks: NTCIR\u2019s Legacy of Research Impact. Springer, Singapore (2021). https:\/\/doi.org\/10.1007\/978-981-15-5554-1","DOI":"10.1007\/978-981-15-5554-1"},{"issue":"3","key":"24_CR35","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1002\/asi.4630210305","volume":"21","author":"G Salton","year":"1970","unstructured":"Salton, G.: Automatic processing of foreign language documents. J. Am. Soc. Inf. Sci. 21(3), 187\u2013194 (1970)","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"24_CR36","unstructured":"Sch\u00e4uble, P., Sheridan, P.: Cross-language information retrieval (CLIR) track overview. In: Voorhees, E.M., Harman, D.K. (eds.) Proceedings of The Sixth Text REtrieval Conference, TREC 1997, Gaithersburg, Maryland, USA, 19\u201321 November 1997. NIST Special Publication, vol. 500\u2013240, pp. 31\u201343. National Institute of Standards and Technology (NIST) (1997). http:\/\/trec.nist.gov\/pubs\/trec6\/papers\/clir_track_US.ps"},{"key":"24_CR37","unstructured":"Smeaton, A.F.: Spanish and Chinese document retrieval in TREC-5. In: Voorhees, E.M., Harman, D.K. (eds.) Proceedings of the Fifth Text REtrieval Conference, TREC 1996, Gaithersburg, Maryland, USA, 20\u201322 November 1996. NIST Special Publication, vol. 500\u2013238. National Institute of Standards and Technology (NIST) (1996). http:\/\/trec.nist.gov\/pubs\/trec5\/papers\/multilingual_track.ps.gz"},{"key":"24_CR38","unstructured":"Thakur, N., Reimers, N., R\u00fcckl\u00e9, A., Srivastava, A., Gurevych, I.: BEIR: a heterogenous benchmark for zero-shot evaluation of information retrieval models. arXiv preprint arXiv:2104.08663 (2021)"},{"issue":"5","key":"24_CR39","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manag. 36(5), 697\u2013716 (2000). https:\/\/doi.org\/10.1016\/S0306-4573(00)00010-8","journal-title":"Inf. Process. Manag."},{"key":"24_CR40","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M.: Coopetition in IR research. In: SIGIR Forum, vol. 54, no. 2, August 2021","DOI":"10.1145\/3483382.3483384"},{"key":"24_CR41","unstructured":"Webber, W., Moffat, A., Zobel, J.: The effect of pooling and evaluation depth on metric stability. In: EVIA@ NTCIR, pp. 7\u201315 (2010)"},{"key":"24_CR42","doi-asserted-by":"crossref","unstructured":"Yang, E., Lewis, D.D., Frieder, O.: On minimizing cost in legal document review workflows. In: Proceedings of the 21st ACM Symposium on Document Engineering, August 2021","DOI":"10.1145\/3469096.3469872"},{"key":"24_CR43","unstructured":"Yarmohammadi, M., et al.: Robust document representations for cross-lingual information retrieval in low-resource settings. In: Proceedings of Machine Translation Summit XVII Volume 1: Research Track, pp. 12\u201320 (2019)"},{"key":"24_CR44","doi-asserted-by":"publisher","unstructured":"Yilmaz, E., Aslam, J.A.: Estimating average precision with incomplete and imperfect judgments. In: Yu, P.S., Tsotras, V.J., Fox, E.A., Liu, B. (eds.) Proceedings of the 2006 ACM CIKM International Conference on Information and Knowledge Management, Arlington, Virginia, USA, 6\u201311 November 2006, pp. 102\u2013111. ACM (2006). https:\/\/doi.org\/10.1145\/1183614.1183633","DOI":"10.1145\/1183614.1183633"},{"key":"24_CR45","doi-asserted-by":"crossref","unstructured":"Zhang, R., et al.: Improving low-resource cross-lingual document retrieval by reranking with deep bilingual representations. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 3173\u20133179 (2019)","DOI":"10.18653\/v1\/P19-1306"},{"key":"24_CR46","doi-asserted-by":"crossref","unstructured":"Zhang, X., Ma, X., Shi, P., Lin, J.: Mr. TyDi: a multi-lingual benchmark for dense retrieval. arXiv preprint arXiv:2108.08787 (2021)","DOI":"10.18653\/v1\/2021.mrl-1.12"},{"key":"24_CR47","doi-asserted-by":"crossref","unstructured":"Zhao, L., Zbib, R., Jiang, Z., Karakos, D., Huang, Z.: Weakly supervised attentional model for low resource ad-hoc cross-lingual information retrieval. In: Proceedings of the 2nd Workshop on Deep Learning Approaches for Low-Resource NLP (DeepLo 2019), pp. 259\u2013264 (2019)","DOI":"10.18653\/v1\/D19-6129"},{"key":"24_CR48","doi-asserted-by":"crossref","unstructured":"Zobel, J.: How reliable are the results of large-scale information retrieval experiments? In: Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 307\u2013314 (1998)","DOI":"10.1145\/290941.291014"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99736-6_24","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T12:58:14Z","timestamp":1710248294000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99736-6_24"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997359","9783030997366"],"references-count":48,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99736-6_24","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}