{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T18:43:35Z","timestamp":1774291415818,"version":"3.50.1"},"publisher-location":"Cham","reference-count":60,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030997359","type":"print"},{"value":"9783030997366","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99736-6_27","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"397-412","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Evaluating the Robustness of Retrieval Pipelines with Query Variation Generators"],"prefix":"10.1007","author":[{"given":"Gustavo","family":"Penha","sequence":"first","affiliation":[]},{"given":"Arthur","family":"C\u00e2mara","sequence":"additional","affiliation":[]},{"given":"Claudia","family":"Hauff","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"27_CR1","doi-asserted-by":"crossref","unstructured":"Abdul-Jaleel, N., et al.: UMass at TREC 2004: novelty and hard. Computer Science Department Faculty Publication Series, p. 189 (2004)","DOI":"10.21236\/ADA460118"},{"key":"27_CR2","doi-asserted-by":"crossref","unstructured":"Alzantot, M., Sharma, Y., Elgohary, A., Ho, B.J., Srivastava, M., Chang, K.W.: Generating natural language adversarial examples. arXiv preprint arXiv:1804.07998 (2018)","DOI":"10.18653\/v1\/D18-1316"},{"key":"27_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1007\/978-3-030-72240-1_14","volume-title":"Advances in Information Retrieval","author":"Y Amemiya","year":"2021","unstructured":"Amemiya, Y., Manabe, T., Fujita, S., Sakai, T.: How do users revise zero-hit product search queries? In: Hiemstra, D., Moens, M.-F., Mothe, J., Perego, R., Potthast, M., Sebastiani, F. (eds.) ECIR 2021. LNCS, vol. 12657, pp. 185\u2013192. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-72240-1_14"},{"key":"27_CR4","doi-asserted-by":"crossref","unstructured":"Bailey, P., Moffat, A., Scholer, F., Thomas, P.: User variability and IR system evaluation. In: Proceedings of The 38th International ACM SIGIR conference on research and development in Information Retrieval, pp. 625\u2013634 (2015)","DOI":"10.1145\/2766462.2767728"},{"key":"27_CR5","doi-asserted-by":"crossref","unstructured":"Bailey, P., Moffat, A., Scholer, F., Thomas, P.: Uqv100: a test collection with query variability. In: Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval, pp. 725\u2013728 (2016)","DOI":"10.1145\/2911451.2914671"},{"key":"27_CR6","doi-asserted-by":"crossref","unstructured":"Bailey, P., Moffat, A., Scholer, F., Thomas, P.: Retrieval consistency in the presence of query variations. In: Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 395\u2013404 (2017)","DOI":"10.1145\/3077136.3080839"},{"key":"27_CR7","doi-asserted-by":"crossref","unstructured":"Belkin, N.J., Cool, C., Croft, W.B., Callan, J.P.: The effect multiple query representations on information retrieval system performance. In: Proceedings of the 16th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 339\u2013346 (1993)","DOI":"10.1145\/160688.160760"},{"issue":"3","key":"27_CR8","doi-asserted-by":"publisher","first-page":"431","DOI":"10.1016\/0306-4573(94)00057-A","volume":"31","author":"NJ Belkin","year":"1995","unstructured":"Belkin, N.J., Kantor, P., Fox, E.A., Shaw, J.A.: Combining the evidence of multiple query representations for information retrieval. Inf. Process. Manage. 31(3), 431\u2013448 (1995)","journal-title":"Inf. Process. Manage."},{"key":"27_CR9","unstructured":"Benham, R., Culpepper, J.S., Gallagher, L., Lu, X., Mackenzie, J.: Towards efficient and effective query variant generation. In: DESIRES, pp. 62\u201367 (2018)"},{"issue":"4","key":"27_CR10","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3345001","volume":"37","author":"R Benham","year":"2019","unstructured":"Benham, R., Mackenzie, J., Moffat, A., Culpepper, J.S.: Boosting search performance using query variations. ACM Trans. Inf. Syst. (TOIS) 37(4), 1\u201325 (2019)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"27_CR11","doi-asserted-by":"crossref","unstructured":"Buckley, C., Walz, J.: The TREC-8 query track. In: TREC (1999)","DOI":"10.6028\/NIST.SP.500-246.query-overview"},{"key":"27_CR12","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"605","DOI":"10.1007\/978-3-030-45439-5_40","volume-title":"Advances in Information Retrieval","author":"A C\u00e2mara","year":"2020","unstructured":"C\u00e2mara, A., Hauff, C.: Diagnosing BERT with retrieval heuristics. In: Jose, J.M., et al. (eds.) ECIR 2020. LNCS, vol. 12035, pp. 605\u2013618. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45439-5_40"},{"key":"27_CR13","doi-asserted-by":"crossref","unstructured":"Chakraborty, A., Ganguly, D., Conlan, O.: Retrievability based document selection for relevance feedback with automatically generated query variants. In: Proceedings of the 29th ACM International Conference on Information & Knowledge Management, pp. 125\u2013134 (2020)","DOI":"10.1145\/3340531.3412032"},{"issue":"1","key":"27_CR14","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Measur. 20(1), 37\u201346 (1960)","journal-title":"Educ. Psychol. Measur."},{"key":"27_CR15","doi-asserted-by":"crossref","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M.: Overview of the TREC 2019 deep learning track. arXiv preprint arXiv:2003.07820 (2020)","DOI":"10.6028\/NIST.SP.1266.deep-overview"},{"key":"27_CR16","doi-asserted-by":"crossref","unstructured":"Dai, Z., Xiong, C., Callan, J., Liu, Z.: Convolutional neural networks for soft-matching N-grams in ad-hoc search. In: Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining, pp. 126\u2013134 (2018)","DOI":"10.1145\/3159652.3159659"},{"key":"27_CR17","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: Bert: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186 (2019)"},{"key":"27_CR18","unstructured":"Fan, A., et al.: Beyond English-centric multilingual machine translation. arXiv preprint arXiv:2010.11125 (2020)"},{"key":"27_CR19","doi-asserted-by":"crossref","unstructured":"Federmann, C., Elachqar, O., Quirk, C.: Multilingual whispers: generating paraphrases with translation. In: Proceedings of the 5th Workshop on Noisy User-generated Text (W-NUT 2019), pp. 17\u201326 (2019)","DOI":"10.18653\/v1\/D19-5503"},{"key":"27_CR20","doi-asserted-by":"crossref","unstructured":"Gao, J., Lanchantin, J., Soffa, M.L., Qi, Y.: Black-box generation of adversarial text sequences to evade deep learning classifiers. In: 2018 IEEE Security and Privacy Workshops (SPW), pp. 50\u201356. IEEE (2018)","DOI":"10.1109\/SPW.2018.00016"},{"key":"27_CR21","unstructured":"Gardner, M., et al.: Evaluating models\u2019 local decision boundaries via contrast sets. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings, pp. 1307\u20131323 (2020)"},{"key":"27_CR22","doi-asserted-by":"crossref","unstructured":"Garg, S., Ramakrishnan, G.: Bae: Bert-based adversarial examples for text classification. arXiv preprint arXiv:2004.01970 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.498"},{"key":"27_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"166","DOI":"10.1007\/978-3-030-45442-5_21","volume-title":"Advances in Information Retrieval","author":"H Hashemi","year":"2020","unstructured":"Hashemi, H., Aliannejadi, M., Zamani, H., Croft, W.B.: ANTIQUE: a non-factoid question answering benchmark. In: Jose, J.M., et al. (eds.) ECIR 2020. LNCS, vol. 12036, pp. 166\u2013173. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45442-5_21"},{"key":"27_CR24","doi-asserted-by":"crossref","unstructured":"Hirsch, S., Guy, I., Nus, A., Dagan, A., Kurland, O.: Query reformulation in e-commerce search. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1319\u20131328 (2020)","DOI":"10.1145\/3397271.3401065"},{"issue":"7","key":"27_CR25","doi-asserted-by":"publisher","first-page":"1358","DOI":"10.1002\/asi.21071","volume":"60","author":"BJ Jansen","year":"2009","unstructured":"Jansen, B.J., Booth, D.L., Spink, A.: Patterns of query reformulation during web searching. J. Am. Soc. Inf. Sci. Technol. 60(7), 1358\u20131371 (2009)","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"27_CR26","doi-asserted-by":"crossref","unstructured":"Jones, R., Rey, B., Madani, O., Greiner, W.: Generating query substitutions. In: Proceedings of the 15th international conference on World Wide Web, pp. 387\u2013396 (2006)","DOI":"10.1145\/1135777.1135835"},{"key":"27_CR27","doi-asserted-by":"crossref","unstructured":"Lin, J., Nogueira, R., Yates, A.: Pretrained transformers for text ranking: Bert and beyond. arXiv preprint arXiv:2010.06467 (2020)","DOI":"10.2200\/S01123ED1V01Y202108HLT053"},{"key":"27_CR28","doi-asserted-by":"crossref","unstructured":"Lu, X., Kurland, O., Culpepper, J.S., Craswell, N., Rom, O.: Relevance modeling with multiple query variations. In: Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval, pp. 27\u201334 (2019)","DOI":"10.1145\/3341981.3344224"},{"key":"27_CR29","doi-asserted-by":"crossref","unstructured":"Ma, X., Santos, C.N.d., Arnold, A.O.: Contrastive fine-tuning improves robustness for neural rankers. arXiv preprint arXiv:2105.12932 (2021)","DOI":"10.18653\/v1\/2021.findings-acl.51"},{"key":"27_CR30","doi-asserted-by":"crossref","unstructured":"MacAvaney, S.: OpenNIR: a complete neural ad-hoc ranking pipeline. In: WSDM 2020 (2020)","DOI":"10.1145\/3336191.3371864"},{"key":"27_CR31","unstructured":"MacAvaney, S., Feldman, S., Goharian, N., Downey, D., Cohan, A.: Abnirml: analyzing the behavior of neural IR models. arXiv preprint arXiv:2011.00696 (2020)"},{"key":"27_CR32","doi-asserted-by":"crossref","unstructured":"MacAvaney, S., Nardini, F.M., Perego, R., Tonellotto, N., Goharian, N., Frieder, O.: Expansion via prediction of importance with contextualization. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1573\u20131576 (2020)","DOI":"10.1145\/3397271.3401262"},{"key":"27_CR33","doi-asserted-by":"crossref","unstructured":"MacAvaney, S., Yates, A., Feldman, S., Downey, D., Cohan, A., Goharian, N.: Simplified data wrangling with ir_datasets. In: SIGIR (2021)","DOI":"10.1145\/3404835.3463254"},{"key":"27_CR34","doi-asserted-by":"crossref","unstructured":"Macdonald, C., Tonellotto, N.: Declarative experimentation in information retrieval using pyterrier. In: Proceedings of ICTIR 2020 (2020)","DOI":"10.1145\/3409256.3409829"},{"key":"27_CR35","doi-asserted-by":"crossref","unstructured":"Mallinson, J., Sennrich, R., Lapata, M.: Paraphrasing revisited with neural machine translation. In: Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers, pp. 881\u2013893 (2017)","DOI":"10.18653\/v1\/E17-1083"},{"key":"27_CR36","doi-asserted-by":"crossref","unstructured":"Moffat, A., Scholer, F., Thomas, P., Bailey, P.: Pooled evaluation over query variations: users are as diverse as systems. In: Proceedings of the 24th ACM International on Conference on Information and Knowledge Management, pp. 1759\u20131762 (2015)","DOI":"10.1145\/2806416.2806606"},{"key":"27_CR37","doi-asserted-by":"crossref","unstructured":"Morris, J., Lifland, E., Yoo, J.Y., Grigsby, J., Jin, D., Qi, Y.: Textattack: a framework for adversarial attacks, data augmentation, and adversarial training in NLP. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pp. 119\u2013126 (2020)","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"27_CR38","doi-asserted-by":"crossref","unstructured":"Mrk\u0161i\u0107, N., et al.: Counter-fitting word vectors to linguistic constraints. arXiv preprint arXiv:1603.00892 (2016)","DOI":"10.18653\/v1\/N16-1018"},{"key":"27_CR39","unstructured":"Nguyen, T., Rosenberg, M., Song, X., Gao, J., Tiwary, S., Majumder, R., Deng, L.: Ms marco: a human generated machine reading comprehension dataset. In: CoCo@ NIPS (2016)"},{"key":"27_CR40","unstructured":"Nogueira, R., Cho, K.: Passage re-ranking with bert. arXiv preprint arXiv:1901.04085 (2019)"},{"key":"27_CR41","doi-asserted-by":"crossref","unstructured":"Nogueira, R., Jiang, Z., Lin, J.: Document ranking with a pretrained sequence-to-sequence model. arXiv preprint arXiv:2003.06713 (2020)","DOI":"10.18653\/v1\/2020.findings-emnlp.63"},{"key":"27_CR42","unstructured":"Padigela, H., Zamani, H., Croft, W.B.: Investigating the successes and failures of Bert for passage re-ranking. arXiv preprint arXiv:1905.01758 (2019)"},{"key":"27_CR43","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"699","DOI":"10.1007\/978-3-030-45439-5_46","volume-title":"Advances in Information Retrieval","author":"G Penha","year":"2020","unstructured":"Penha, G., Hauff, C.: Curriculum learning strategies for IR. In: Jose, J.M., Yilmaz, E., Magalh\u00e3es, J., Castells, P., Ferro, N., Silva, M.J., Martins, F. (eds.) ECIR 2020. LNCS, vol. 12035, pp. 699\u2013713. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45439-5_46"},{"key":"27_CR44","doi-asserted-by":"crossref","unstructured":"Pham, T.M., Bui, T., Mai, L., Nguyen, A.: Out of order: How important is the sequential order of words in a sentence in natural language understanding tasks? arXiv preprint arXiv:2012.15180 (2020)","DOI":"10.18653\/v1\/2021.findings-acl.98"},{"key":"27_CR45","unstructured":"Qiao, Y., Xiong, C., Liu, Z., Liu, Z.: Understanding the behaviors of Bert in ranking. arXiv preprint arXiv:1904.07531 (2019)"},{"key":"27_CR46","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. arXiv preprint arXiv:1910.10683 (2019)"},{"key":"27_CR47","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"489","DOI":"10.1007\/978-3-030-15712-8_32","volume-title":"Advances in Information Retrieval","author":"D Rennings","year":"2019","unstructured":"Rennings, D., Moraes, F., Hauff, C.: An axiomatic approach to diagnosing neural IR models. In: Azzopardi, L., Stein, B., Fuhr, N., Mayr, P., Hauff, C., Hiemstra, D. (eds.) ECIR 2019. LNCS, vol. 11437, pp. 489\u2013503. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-15712-8_32"},{"key":"27_CR48","doi-asserted-by":"crossref","unstructured":"Ribeiro, M.T., Wu, T., Guestrin, C., Singh, S.: Beyond accuracy: behavioral testing of NLP models with checklist. arXiv preprint arXiv:2005.04118 (2020)","DOI":"10.24963\/ijcai.2021\/659"},{"key":"27_CR49","doi-asserted-by":"publisher","unstructured":"Robertson, S.E., Walker, S.: Some simple effective approximations to the 2-Poisson model for probabilistic weighted retrieval. In: SIGIR 1994, pp. 232\u2013241. Springer, Cham (1994). https:\/\/doi.org\/10.1007\/978-1-4471-2099-5_24","DOI":"10.1007\/978-1-4471-2099-5_24"},{"key":"27_CR50","unstructured":"Spark-Jones, K.: Report on the need for and provision of an \u2018ideal\u2019 information retrieval test collection. Computer Laboratory (1975)"},{"key":"27_CR51","unstructured":"Wolf, T., et al.: Transformers: state-of-the-art natural language processing. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pp. 38\u201345. Association for Computational Linguistics, Online, October 2020. https:\/\/www.aclweb.org\/anthology\/2020.emnlp-demos.6"},{"issue":"1","key":"27_CR52","first-page":"1","volume":"4","author":"JR Wood","year":"2008","unstructured":"Wood, J.R., Wood, L.E.: Card sorting: current practices and beyond. J. Usabil. Stud. 4(1), 1\u20136 (2008)","journal-title":"J. Usabil. Stud."},{"key":"27_CR53","unstructured":"Wu, C., Zhang, R., Guo, J., Fan, Y., Cheng, X.: Are neural ranking models robust? arXiv preprint arXiv:2108.05018 (2021)"},{"key":"27_CR54","doi-asserted-by":"crossref","unstructured":"Xiong, C., Dai, Z., Callan, J., Liu, Z., Power, R.: End-to-end neural ad-hoc ranking with kernel pooling. In: Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 55\u201364 (2017)","DOI":"10.1145\/3077136.3080809"},{"key":"27_CR55","unstructured":"Yang, W., Zhang, H., Lin, J.: Simple applications of Bert for ad hoc document retrieval. arXiv preprint arXiv:1903.10972 (2019)"},{"key":"27_CR56","doi-asserted-by":"crossref","unstructured":"Yang, Y., Zhang, Y., Tar, C., Baldridge, J.: PAWS-X: a cross-lingual adversarial dataset for paraphrase identification. In: Proceedings of EMNLP (2019)","DOI":"10.18653\/v1\/D19-1382"},{"key":"27_CR57","doi-asserted-by":"crossref","unstructured":"Zendel, O., Shtok, A., Raiber, F., Kurland, O., Culpepper, J.S.: Information needs, queries, and query performance prediction. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 395\u2013404 (2019)","DOI":"10.1145\/3331184.3331253"},{"key":"27_CR58","doi-asserted-by":"crossref","unstructured":"Zhan, J., Mao, J., Liu, Y., Zhang, M., Ma, S.: An analysis of Bert in document ranking. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1941\u20131944 (2020)","DOI":"10.1145\/3397271.3401325"},{"key":"27_CR59","doi-asserted-by":"crossref","unstructured":"Zhuang, S., Zuccon, G.: Dealing with typos for Bert-based passage retrieval and ranking. arXiv preprint arXiv:2108.12139 (2021)","DOI":"10.18653\/v1\/2021.emnlp-main.225"},{"key":"27_CR60","doi-asserted-by":"crossref","unstructured":"Zuccon, G., Palotti, J., Hanbury, A.: Query variations and their effect on comparing information retrieval systems. In: Proceedings of the 25th ACM International on Conference on Information and Knowledge Management, pp. 691\u2013700 (2016)","DOI":"10.1145\/2983323.2983723"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99736-6_27","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,21]],"date-time":"2024-09-21T15:43:04Z","timestamp":1726933384000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99736-6_27"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997359","9783030997366"],"references-count":60,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99736-6_27","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}